@cogitator-ai/evals 0.1.0

package/dist/eval-suite.js

@@ -0,0 +1,230 @@
+import { writeFileSync } from 'node:fs';
+import { EvalSuiteConfigSchema } from './schema';
+import { bindJudgeContext } from './metrics/llm-judge';
+import { aggregate } from './stats';
+import { report } from './reporters';
+function isLLMMetric(m) {
+    return 'requiresJudge' in m && m.requiresJudge === true;
+}
+export class EvalSuite {
+    dataset;
+    target;
+    boundMetrics;
+    statisticalMetrics;
+    assertionFns;
+    concurrency;
+    timeout;
+    retries;
+    onProgress;
+    constructor(opts) {
+        const config = EvalSuiteConfigSchema.parse({
+            concurrency: opts.concurrency,
+            timeout: opts.timeout,
+            retries: opts.retries,
+        });
+        this.dataset = opts.dataset;
+        this.target = opts.target;
+        this.statisticalMetrics = opts.statisticalMetrics ?? [];
+        this.assertionFns = opts.assertions ?? [];
+        this.concurrency = config.concurrency;
+        this.timeout = config.timeout;
+        this.retries = config.retries;
+        this.onProgress = opts.onProgress;
+        this.validateTarget();
+        const rawMetrics = opts.metrics ?? [];
+        const hasLLMMetrics = rawMetrics.some(isLLMMetric);
+        if (hasLLMMetrics && !opts.judge) {
+            throw new Error('LLM metrics require a judge config');
+        }
+        this.boundMetrics = rawMetrics.map((m) => {
+            if (isLLMMetric(m) && opts.judge) {
+                return bindJudgeContext(m, {
+                    cogitator: { run: async ({ input }) => ({ output: input }) },
+                    judgeConfig: opts.judge,
+                });
+            }
+            return m;
+        });
+    }
+    validateTarget() {
+        const { fn, agent, cogitator } = this.target;
+        const hasFn = fn !== undefined;
+        const hasAgent = agent !== undefined;
+        const hasCogitator = cogitator !== undefined;
+        if (hasFn && (hasAgent || hasCogitator)) {
+            throw new Error('Target must have either fn or agent+cogitator, not both');
+        }
+        if (!hasFn && !hasAgent && !hasCogitator) {
+            throw new Error('Target must have either fn or agent+cogitator');
+        }
+        if (hasAgent && !hasCogitator) {
+            throw new Error('Agent target requires cogitator instance');
+        }
+        if (hasCogitator && !hasAgent) {
+            throw new Error('Cogitator target requires agent instance');
+        }
+    }
+    async run() {
+        const suiteStart = Date.now();
+        const cases = [...this.dataset.cases];
+        const total = cases.length;
+        let completed = 0;
+        const indexed = [];
+        let active = 0;
+        let nextIdx = 0;
+        if (total > 0) {
+            await new Promise((resolve) => {
+                const drain = () => {
+                    while (active < this.concurrency && nextIdx < total) {
+                        const i = nextIdx++;
+                        const evalCase = cases[i];
+                        active++;
+                        this.executeCase(evalCase)
+                            .then((caseResult) => this.evaluateCaseMetrics(caseResult).then((scores) => ({ ...caseResult, scores })))
+                            .then((scored) => {
+                            indexed.push({ idx: i, result: scored });
+                            completed++;
+                            this.onProgress?.({ completed, total, currentCase: evalCase });
+                        })
+                            .finally(() => {
+                            active--;
+                            if (nextIdx >= total && active === 0) {
+                                resolve();
+                            }
+                            else {
+                                drain();
+                            }
+                        });
+                    }
+                };
+                drain();
+            });
+        }
+        indexed.sort((a, b) => a.idx - b.idx);
+        const orderedResults = indexed.map((e) => e.result);
+        const aggregated = this.aggregateScores(orderedResults);
+        for (const statMetric of this.statisticalMetrics) {
+            const score = statMetric(orderedResults);
+            aggregated[score.name] = { name: score.name, ...aggregate([score.score]) };
+        }
+        const totalCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
+        const suiteDuration = Date.now() - suiteStart;
+        const stats = { total, duration: suiteDuration, cost: totalCost };
+        const assertionResults = this.assertionFns.map((fn) => fn(aggregated, stats));
+        const suiteResult = {
+            results: orderedResults,
+            aggregated,
+            assertions: assertionResults,
+            stats,
+            report: (type, options) => {
+                report({
+                    results: orderedResults.map((r) => ({
+                        case: { input: r.case.input, expected: r.case.expected },
+                        output: r.output,
+                        duration: r.duration,
+                        scores: r.scores,
+                    })),
+                    aggregated,
+                    assertions: assertionResults,
+                    stats,
+                }, type, options);
+            },
+            saveBaseline: (path) => {
+                const baseline = {};
+                for (const [name, agg] of Object.entries(aggregated)) {
+                    baseline[name] = agg.mean;
+                }
+                writeFileSync(path, JSON.stringify(baseline, null, 2));
+            },
+        };
+        return suiteResult;
+    }
+    async executeCase(evalCase) {
+        for (let attempt = 0; attempt <= this.retries; attempt++) {
+            try {
+                return await this.executeCaseAttempt(evalCase);
+            }
+            catch {
+                if (attempt < this.retries)
+                    continue;
+            }
+        }
+        return {
+            case: evalCase,
+            output: '',
+            duration: 0,
+        };
+    }
+    async executeCaseAttempt(evalCase) {
+        const start = Date.now();
+        const work = this.target.fn
+            ? this.executeFnTarget(evalCase)
+            : this.executeAgentTarget(evalCase);
+        const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => reject(new Error('__timeout__')), this.timeout);
+        });
+        try {
+            return await Promise.race([work, timeoutPromise]);
+        }
+        catch (err) {
+            if (err.message === '__timeout__') {
+                return {
+                    case: evalCase,
+                    output: '',
+                    duration: Date.now() - start,
+                };
+            }
+            throw err;
+        }
+    }
+    async executeFnTarget(evalCase) {
+        const start = Date.now();
+        const output = await this.target.fn(evalCase.input);
+        return { case: evalCase, output, duration: Date.now() - start };
+    }
+    async executeAgentTarget(evalCase) {
+        const start = Date.now();
+        const cogitator = this.target.cogitator;
+        const runResult = await cogitator.run(this.target.agent, {
+            input: evalCase.input,
+            context: evalCase.context,
+        });
+        const duration = Date.now() - start;
+        const result = {
+            case: evalCase,
+            output: runResult.output ?? '',
+            duration,
+        };
+        if (runResult.usage) {
+            result.usage = runResult.usage;
+        }
+        if (runResult.toolCalls) {
+            result.toolCalls = runResult.toolCalls;
+        }
+        return result;
+    }
+    async evaluateCaseMetrics(result) {
+        if (this.boundMetrics.length === 0)
+            return [];
+        return Promise.all(this.boundMetrics.map((m) => m(result)));
+    }
+    aggregateScores(results) {
+        const scoresByMetric = new Map();
+        for (const r of results) {
+            for (const s of r.scores) {
+                let arr = scoresByMetric.get(s.name);
+                if (!arr) {
+                    arr = [];
+                    scoresByMetric.set(s.name, arr);
+                }
+                arr.push(s.score);
+            }
+        }
+        const aggregated = {};
+        for (const [name, values] of scoresByMetric) {
+            aggregated[name] = { name, ...aggregate(values) };
+        }
+        return aggregated;
+    }
+}
+//# sourceMappingURL=eval-suite.js.map

package/dist/eval-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAqCrC,SAAS,WAAW,CAAC,CAAW;IAC9B,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACvC,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACjC,OAAO,gBAAgB,CAAC,CAAC,EAAE;oBACzB,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAqB,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;oBAC/E,WAAW,EAAE,IAAI,CAAC,KAAK;iBACxB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BACvB,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC7E,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO;oBAAE,SAAS;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAAa,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;gBAC7C,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAC7B,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,31 @@
+export declare const VERSION = "0.1.0";
+export { EvalSuite } from './eval-suite';
+export type { EvalTarget, EvalProgress, EvalSuiteOptions, EvalSuiteResult } from './eval-suite';
+export { EvalComparison } from './eval-comparison';
+export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison';
+export { EvalBuilder } from './eval-builder';
+export { Dataset } from './datasets';
+export { loadJsonl } from './datasets';
+export { loadCsv } from './datasets';
+export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge';
+export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge';
+export { latency, cost, tokenUsage } from './metrics/statistical';
+export { metric } from './metrics/custom';
+export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types';
+export type { CustomMetricConfig } from './metrics/custom';
+export { threshold } from './assertions';
+export { noRegression } from './assertions';
+export { assertion } from './assertions';
+export type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions';
+export { report } from './reporters';
+export type { ReporterType, ReporterOptions } from './reporters';
+export { pairedTTest } from './stats/t-test';
+export type { TTestResult } from './stats/t-test';
+export { mcnemarsTest } from './stats/mcnemar';
+export type { McNemarResult } from './stats/mcnemar';
+export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles';
+export type { EvalCase, EvalSuiteConfig, JudgeConfig, EvalComparisonConfig } from './schema';
+export { createRunEvalTool, evalTools } from './tools';
+export type { EvalTool } from './tools';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClG,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAE7F,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -0,0 +1,20 @@
+export const VERSION = '0.1.0';
+export { EvalSuite } from './eval-suite';
+export { EvalComparison } from './eval-comparison';
+export { EvalBuilder } from './eval-builder';
+export { Dataset } from './datasets';
+export { loadJsonl } from './datasets';
+export { loadCsv } from './datasets';
+export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge';
+export { latency, cost, tokenUsage } from './metrics/statistical';
+export { metric } from './metrics/custom';
+export { threshold } from './assertions';
+export { noRegression } from './assertions';
+export { assertion } from './assertions';
+export { report } from './reporters';
+export { pairedTTest } from './stats/t-test';
+export { mcnemarsTest } from './stats/mcnemar';
+export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles';
+export { createRunEvalTool, evalTools } from './tools';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/custom.d.ts

@@ -0,0 +1,18 @@
+import type { MetricFn } from './types';
+export interface CustomMetricConfig {
+    name: string;
+    evaluate: (data: {
+        input: string;
+        output: string;
+        expected?: string;
+        context?: Record<string, unknown>;
+    }) => Promise<{
+        score: number;
+        details?: string;
+    }> | {
+        score: number;
+        details?: string;
+    };
+}
+export declare function metric(config: CustomMetricConfig): MetricFn;
+//# sourceMappingURL=custom.d.ts.map

package/dist/metrics/custom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,SAAS,CAAC;AAExD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CAAC,IAAI,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CA4B3D"}

package/dist/metrics/custom.js

@@ -0,0 +1,28 @@
+export function metric(config) {
+    const fn = (async (result) => {
+        try {
+            const { score, details } = await config.evaluate({
+                input: result.case.input,
+                output: result.output,
+                expected: result.case.expected,
+                context: result.case.context,
+            });
+            const clamped = Math.max(0, Math.min(1, score));
+            return {
+                name: config.name,
+                score: clamped,
+                ...(details !== undefined && { details }),
+            };
+        }
+        catch (err) {
+            return {
+                name: config.name,
+                score: 0,
+                details: `evaluate error: ${err.message}`,
+            };
+        }
+    });
+    fn.metricName = config.name;
+    return fn;
+}
+//# sourceMappingURL=custom.js.map

package/dist/metrics/custom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAYA,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAEhD,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAoB,GAAa,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -0,0 +1,11 @@
+import type { ZodType } from 'zod';
+import type { MetricFn } from './types';
+interface MatchOptions {
+    caseSensitive?: boolean;
+}
+export declare function exactMatch(opts?: MatchOptions): MetricFn;
+export declare function contains(opts?: MatchOptions): MetricFn;
+export declare function regex(pattern: string | RegExp): MetricFn;
+export declare function jsonSchema(schema: ZodType): MetricFn;
+export {};
+//# sourceMappingURL=deterministic.d.ts.map

package/dist/metrics/deterministic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,UAAU,YAAY;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAmBxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAmBtD;AAED,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAWxD;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,GAAG,QAAQ,CAwBpD"}

package/dist/metrics/deterministic.js

@@ -0,0 +1,74 @@
+function createMetricFn(name, fn) {
+    fn.metricName = name;
+    return fn;
+}
+export function exactMatch(opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    return createMetricFn('exactMatch', (async (result) => {
+        const expected = result.case.expected;
+        if (expected === undefined) {
+            return { name: 'exactMatch', score: 0, details: 'no expected value provided' };
+        }
+        const output = result.output.trim();
+        const target = expected.trim();
+        const match = caseSensitive ? output === target : output.toLowerCase() === target.toLowerCase();
+        return {
+            name: 'exactMatch',
+            score: match ? 1 : 0,
+            details: match ? undefined : `expected "${target}", got "${output}"`,
+        };
+    }));
+}
+export function contains(opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    return createMetricFn('contains', (async (result) => {
+        const expected = result.case.expected;
+        if (expected === undefined) {
+            return { name: 'contains', score: 0, details: 'no expected value provided' };
+        }
+        const output = caseSensitive ? result.output : result.output.toLowerCase();
+        const target = caseSensitive ? expected : expected.toLowerCase();
+        const found = output.includes(target);
+        return {
+            name: 'contains',
+            score: found ? 1 : 0,
+            details: found ? undefined : `output does not contain "${expected}"`,
+        };
+    }));
+}
+export function regex(pattern) {
+    const re = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
+    return createMetricFn('regex', (async (result) => {
+        const match = re.test(result.output);
+        return {
+            name: 'regex',
+            score: match ? 1 : 0,
+            details: match ? undefined : `output does not match pattern ${re}`,
+        };
+    }));
+}
+export function jsonSchema(schema) {
+    return createMetricFn('jsonSchema', (async (result) => {
+        let parsed;
+        try {
+            parsed = JSON.parse(result.output);
+        }
+        catch (e) {
+            return {
+                name: 'jsonSchema',
+                score: 0,
+                details: `invalid JSON: ${e.message}`,
+            };
+        }
+        const validation = schema.safeParse(parsed);
+        if (validation.success) {
+            return { name: 'jsonSchema', score: 1 };
+        }
+        return {
+            name: 'jsonSchema',
+            score: 0,
+            details: `schema validation failed: ${validation.error.message}`,
+        };
+    }));
+}
+//# sourceMappingURL=deterministic.js.map

package/dist/metrics/deterministic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IAEnD,OAAO,cAAc,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACjF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IAEnD,OAAO,cAAc,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QAC/E,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,OAAwB;IAC5C,MAAM,EAAE,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEvE,OAAO,cAAc,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC/C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,EAAE,EAAE;SACnE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe;IACxC,OAAO,cAAc,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACpD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC1C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/index.d.ts

@@ -0,0 +1,8 @@
+export type { MetricScore, EvalCaseResult, MetricFn, StatisticalMetricFn } from './types';
+export { exactMatch, contains, regex, jsonSchema } from './deterministic';
+export { metric } from './custom';
+export type { CustomMetricConfig } from './custom';
+export { latency, cost, tokenUsage } from './statistical';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge';
+export type { JudgeContext, LLMMetricFn } from './llm-judge';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}

package/dist/metrics/index.js

@@ -0,0 +1,5 @@
+export { exactMatch, contains, regex, jsonSchema } from './deterministic';
+export { metric } from './custom';
+export { latency, cost, tokenUsage } from './statistical';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge';
+//# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}

package/dist/metrics/llm-judge.d.ts

@@ -0,0 +1,27 @@
+import type { JudgeConfig } from '../schema';
+import type { MetricFn } from './types';
+export interface JudgeContext {
+    cogitator: {
+        run: (opts: {
+            input: string;
+        }) => Promise<{
+            output: string;
+        }>;
+    };
+    judgeConfig: JudgeConfig;
+}
+export interface LLMMetricFn extends MetricFn {
+    readonly requiresJudge: true;
+    readonly __judgeSystemPrompt: string;
+    readonly __judgeName: string;
+}
+export declare function bindJudgeContext(metric: LLMMetricFn, context: JudgeContext): MetricFn;
+export declare function faithfulness(): LLMMetricFn;
+export declare function relevance(): LLMMetricFn;
+export declare function coherence(): LLMMetricFn;
+export declare function helpfulness(): LLMMetricFn;
+export declare function llmMetric(opts: {
+    name: string;
+    prompt: string;
+}): LLMMetricFn;
+//# sourceMappingURL=llm-judge.d.ts.map

package/dist/metrics/llm-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,QAAQ,EAA+B,MAAM,SAAS,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC7E,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsCD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CAmCrF;AAED,wBAAgB,YAAY,IAAI,WAAW,CAK1C;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,WAAW,IAAI,WAAW,CAKzC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}

package/dist/metrics/llm-judge.js

@@ -0,0 +1,77 @@
+const SCORE_REGEX = /\b(0(?:\.\d+)?|1(?:\.0+)?)\b/;
+function parseJudgeOutput(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.score === 'number') {
+            return { score: parsed.score, reasoning: parsed.reasoning };
+        }
+        return null;
+    }
+    catch {
+        const match = raw.match(SCORE_REGEX);
+        if (match) {
+            return { score: parseFloat(match[1]) };
+        }
+        return null;
+    }
+}
+function createJudgeMetric(name, systemPrompt) {
+    const fn = (async (_result) => {
+        return { name, score: 0, details: 'unbound judge metric — call bindJudgeContext first' };
+    });
+    Object.defineProperty(fn, 'metricName', { value: name, writable: false });
+    Object.defineProperty(fn, 'requiresJudge', { value: true, writable: false });
+    Object.defineProperty(fn, '__judgeSystemPrompt', { value: systemPrompt, writable: false });
+    Object.defineProperty(fn, '__judgeName', { value: name, writable: false });
+    return fn;
+}
+function buildUserMessage(result) {
+    const expected = result.case.expected || 'N/A';
+    return `Input: ${result.case.input}\nExpected: ${expected}\nResponse: ${result.output}`;
+}
+export function bindJudgeContext(metric, context) {
+    const name = metric.__judgeName;
+    const systemPrompt = metric.__judgeSystemPrompt;
+    const bound = (async (result) => {
+        try {
+            const userMessage = buildUserMessage(result);
+            const prompt = `${systemPrompt}\n\n${userMessage}`;
+            const runResult = await context.cogitator.run({ input: prompt });
+            const parsed = parseJudgeOutput(runResult.output);
+            if (!parsed) {
+                return { name, score: 0, details: 'could not parse judge response' };
+            }
+            const clamped = Math.max(0, Math.min(1, parsed.score));
+            return {
+                name,
+                score: clamped,
+                ...(parsed.reasoning !== undefined && { details: parsed.reasoning }),
+            };
+        }
+        catch (err) {
+            return {
+                name,
+                score: 0,
+                details: `judge error: ${err.message}`,
+            };
+        }
+    });
+    Object.defineProperty(bound, 'metricName', { value: name, writable: false });
+    return bound;
+}
+export function faithfulness() {
+    return createJudgeMetric('faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input and the response, rate how faithful the response is to the facts and information in the input.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+}
+export function relevance() {
+    return createJudgeMetric('relevance', 'You are evaluating the relevance of an AI assistant\'s response.\n\nGiven the input and the response, rate how relevant the response is to the question asked.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+}
+export function coherence() {
+    return createJudgeMetric('coherence', 'You are evaluating the coherence of an AI assistant\'s response.\n\nGiven the input and the response, rate how coherent, logical, and well-structured the response is.\n\nScore from 0.0 (completely incoherent) to 1.0 (perfectly coherent).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+}
+export function helpfulness() {
+    return createJudgeMetric('helpfulness', 'You are evaluating the helpfulness of an AI assistant\'s response.\n\nGiven the input and the response, rate how helpful and useful the response would be to the user.\n\nScore from 0.0 (completely unhelpful) to 1.0 (perfectly helpful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+}
+export function llmMetric(opts) {
+    return createJudgeMetric(opts.name, `${opts.prompt}\n\nScore from 0.0 to 1.0.\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}`);
+}
+//# sourceMappingURL=llm-judge.js.map

package/dist/metrics/llm-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,GAAG,8BAA8B,CAAC;AAEnD,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACrC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;aACrE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAiB,GAAa,CAAC,OAAO,EAAE;aAClD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,iBAAiB,CACtB,cAAc,EACd,sUAAsU,CACvU,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,iBAAiB,CACtB,aAAa,EACb,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}

package/dist/metrics/statistical.d.ts

@@ -0,0 +1,5 @@
+import type { StatisticalMetricFn } from './types';
+export declare function latency(): StatisticalMetricFn;
+export declare function cost(): StatisticalMetricFn;
+export declare function tokenUsage(): StatisticalMetricFn;
+//# sourceMappingURL=statistical.d.ts.map

package/dist/metrics/statistical.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAYhF,wBAAgB,OAAO,IAAI,mBAAmB,CAmB7C;AAED,wBAAgB,IAAI,IAAI,mBAAmB,CA8B1C;AAED,wBAAgB,UAAU,IAAI,mBAAmB,CAkChD"}