@cogitator-ai/evals 0.5.0 → 0.6.0

package/dist/eval-suite.js

@@ -1,11 +1,72 @@
 import { writeFileSync } from 'node:fs';
 import { EvalSuiteConfigSchema, JudgeConfigSchema } from './schema.js';
+import { nonFiniteScoreError } from './metrics/types.js';
 import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
 import { aggregate } from './stats/index.js';
 import { report } from './reporters/index.js';
 export function isLLMMetric(m) {
     return 'requiresJudge' in m && m.requiresJudge === true;
 }
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function finite(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+function targetUsageOf(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    const usage = value;
+    if (typeof usage.cost !== 'number' || !Number.isFinite(usage.cost))
+        return undefined;
+    return {
+        inputTokens: finite(usage.inputTokens),
+        outputTokens: finite(usage.outputTokens),
+        totalTokens: finite(usage.totalTokens),
+        cost: usage.cost,
+        duration: finite(usage.duration),
+    };
+}
+function addUsage(total, more) {
+    if (!more)
+        return total;
+    if (!total)
+        return more;
+    return {
+        inputTokens: total.inputTokens + more.inputTokens,
+        outputTokens: total.outputTokens + more.outputTokens,
+        totalTokens: total.totalTokens + more.totalTokens,
+        cost: total.cost + more.cost,
+        duration: total.duration + more.duration,
+    };
+}
+/** Resolves when `signal` aborts; `cancel` drops the listener */
+function whenAborted(signal) {
+    let cancel = () => { };
+    const aborted = new Promise((resolve) => {
+        if (signal.aborted) {
+            resolve('aborted');
+            return;
+        }
+        const onAbort = () => resolve('aborted');
+        signal.addEventListener('abort', onAbort, { once: true });
+        cancel = () => signal.removeEventListener('abort', onAbort);
+    });
+    return { aborted, cancel };
+}
+/** Waits for `promise` at most `ms` milliseconds; undefined when it did not settle in time */
+async function waitAtMost(promise, ms) {
+    let timer;
+    const expired = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(undefined), ms);
+    });
+    try {
+        return await Promise.race([promise, expired]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 export class EvalSuite {
     dataset;
     target;
@@ -32,6 +93,7 @@ export class EvalSuite {
         this.onProgress = opts.onProgress;
         this.validateTarget();
         const rawMetrics = opts.metrics ?? [];
+        this.validateMetricNames(rawMetrics);
         const hasLLMMetrics = rawMetrics.some(isLLMMetric);
         if (hasLLMMetrics && !opts.judge) {
             throw new Error('LLM metrics require a judge config');
@@ -62,48 +124,42 @@ export class EvalSuite {
             throw new Error('Cogitator target requires agent instance');
         }
     }
+    /** Two metrics with one name would be aggregated, compared and reported as one */
+    validateMetricNames(metrics) {
+        const seen = new Set();
+        for (const m of [...metrics, ...this.statisticalMetrics]) {
+            const name = m.metricName;
+            if (name === undefined)
+                continue;
+            if (seen.has(name)) {
+                throw new Error(`Two metrics are named '${name}': give each its own name, such as regex(pattern, { name: 'hasDate' })`);
+            }
+            seen.add(name);
+        }
+    }
     async run(options = {}) {
-        const { maxCases } = options;
+        const { maxCases, signal } = options;
         if (maxCases !== undefined && (!Number.isInteger(maxCases) || maxCases < 1)) {
             throw new Error(`maxCases must be a positive integer, got ${maxCases}`);
         }
+        signal?.throwIfAborted();
         const suiteStart = Date.now();
         const cases = this.dataset.cases.slice(0, maxCases);
         const total = cases.length;
-        let completed = 0;
-        const indexed = [];
-        let active = 0;
+        const orderedResults = new Array(total);
         let nextIdx = 0;
-        if (total > 0) {
-            await new Promise((resolve) => {
-                const drain = () => {
-                    while (active < this.concurrency && nextIdx < total) {
-                        const i = nextIdx++;
-                        const evalCase = cases[i];
-                        active++;
-                        void this.executeCase(evalCase)
-                            .then((caseResult) => this.evaluateCaseMetrics(caseResult).then((scores) => ({ ...caseResult, scores })))
-                            .then((scored) => {
-                            indexed.push({ idx: i, result: scored });
-                            completed++;
-                            this.onProgress?.({ completed, total, currentCase: evalCase });
-                        })
-                            .finally(() => {
-                            active--;
-                            if (nextIdx >= total && active === 0) {
-                                resolve();
-                            }
-                            else {
-                                drain();
-                            }
-                        });
-                    }
-                };
-                drain();
-            });
-        }
-        indexed.sort((a, b) => a.idx - b.idx);
-        const orderedResults = indexed.map((e) => e.result);
+        let completed = 0;
+        const worker = async () => {
+            while (nextIdx < total && !signal?.aborted) {
+                const i = nextIdx++;
+                const evalCase = cases[i];
+                orderedResults[i] = await this.runCase(evalCase, signal);
+                completed++;
+                this.reportProgress({ completed, total, currentCase: evalCase });
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(this.concurrency, total) }, worker));
+        signal?.throwIfAborted();
         const aggregated = this.aggregateScores(orderedResults);
         for (const statMetric of this.statisticalMetrics) {
             const score = statMetric(orderedResults);
@@ -113,9 +169,17 @@ export class EvalSuite {
                 ...(score.metadata && { metadata: score.metadata }),
             };
         }
-        const totalCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
-        const suiteDuration = Date.now() - suiteStart;
-        const stats = { total, duration: suiteDuration, cost: totalCost };
+        const targetCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
+        const judgeCost = orderedResults.reduce((sum, r) => sum + r.scores.reduce((s, score) => s + (score.usage?.cost ?? 0), 0), 0);
+        const stats = {
+            total,
+            errors: orderedResults.filter((r) => r.error !== undefined).length,
+            metricErrors: orderedResults.reduce((n, r) => n + r.scores.filter((s) => s.error !== undefined).length, 0),
+            duration: Date.now() - suiteStart,
+            cost: targetCost + judgeCost,
+            targetCost,
+            judgeCost,
+        };
         const assertionResults = this.assertionFns.map((fn) => fn(aggregated, stats));
         const suiteResult = {
             results: orderedResults,
@@ -128,6 +192,7 @@ export class EvalSuite {
                         case: { input: r.case.input, expected: r.case.expected },
                         output: r.output,
                         duration: r.duration,
+                        ...(r.error !== undefined && { error: r.error }),
                         scores: r.scores,
                     })),
                     aggregated,
@@ -137,58 +202,106 @@ export class EvalSuite {
             },
             saveBaseline: (path) => {
                 const baseline = {};
+                const invalid = [];
                 for (const [name, agg] of Object.entries(aggregated)) {
-                    baseline[name] = agg.mean;
+                    if (Number.isFinite(agg.mean))
+                        baseline[name] = agg.mean;
+                    else
+                        invalid.push(`${name} = ${agg.mean}`);
+                }
+                if (invalid.length > 0) {
+                    throw new Error(`Cannot save a baseline with values that are not finite numbers: ${invalid.join(', ')}`);
                 }
                 writeFileSync(path, JSON.stringify(baseline, null, 2));
             },
         };
         return suiteResult;
     }
-    async executeCase(evalCase) {
+    reportProgress(progress) {
+        if (!this.onProgress)
+            return;
+        const warn = (error) => process.emitWarning(`EvalSuite onProgress threw: ${errorMessage(error)}`, {
+            code: 'COGITATOR_EVALS_ON_PROGRESS',
+        });
+        try {
+            const returned = this.onProgress(progress);
+            if (returned instanceof Promise)
+                returned.catch(warn);
+        }
+        catch (error) {
+            warn(error);
+        }
+    }
+    async runCase(evalCase, signal) {
+        const caseResult = await this.executeCase(evalCase, signal);
+        if (signal?.aborted)
+            return { ...caseResult, scores: [] };
+        return { ...caseResult, scores: await this.evaluateCaseMetrics(caseResult) };
+    }
+    async executeCase(evalCase, signal) {
         const start = Date.now();
         let lastError;
-        for (let attempt = 0; attempt <= this.retries; attempt++) {
-            try {
-                return await this.executeCaseAttempt(evalCase);
-            }
-            catch (err) {
-                lastError = err;
+        let usage;
+        let attempts = 0;
+        for (let attempt = 0; attempt <= this.retries && !signal?.aborted; attempt++) {
+            attempts++;
+            const outcome = await this.executeCaseAttempt(evalCase, attempt, signal);
+            if (outcome.ok) {
+                const total = addUsage(usage, outcome.result.usage);
+                return { ...outcome.result, attempts, ...(total && { usage: total }) };
             }
+            usage = addUsage(usage, outcome.usage);
+            lastError = outcome.error;
         }
         return {
             case: evalCase,
             output: '',
             duration: Date.now() - start,
-            error: lastError instanceof Error ? lastError.message : String(lastError),
+            error: errorMessage(signal?.aborted ? signal.reason : lastError),
+            attempts,
+            ...(usage && { usage }),
         };
     }
-    async executeCaseAttempt(evalCase) {
-        const work = this.target.fn
-            ? this.executeFnTarget(evalCase)
-            : this.executeAgentTarget(evalCase);
-        let timer;
-        const timeoutPromise = new Promise((_, reject) => {
-            timer = setTimeout(() => reject(new Error(`Timed out after ${this.timeout}ms`)), this.timeout);
-        });
+    /**
+     * One attempt, aborted through its signal after `timeout`. An aborted attempt gets as long
+     * again to stop before the case moves on, so a target that honors the signal never overlaps
+     * its retry and the concurrency limit holds. Usage of an attempt that still finished counts.
+     */
+    async executeCaseAttempt(evalCase, attempt, suiteSignal) {
+        const controller = new AbortController();
+        const signal = suiteSignal
+            ? AbortSignal.any([suiteSignal, controller.signal])
+            : controller.signal;
+        const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${this.timeout}ms`)), this.timeout);
+        const settled = (async () => this.target.fn
+            ? this.executeFnTarget(evalCase, { signal, case: evalCase, attempt })
+            : this.executeAgentTarget(evalCase, signal))().then((result) => ({ ok: true, result }), (error) => ({ ok: false, error }));
+        const abort = whenAborted(signal);
         try {
-            return await Promise.race([work, timeoutPromise]);
+            const first = await Promise.race([settled, abort.aborted]);
+            if (first !== 'aborted' && (first.ok || !signal.aborted))
+                return first;
+            const late = await waitAtMost(settled, this.timeout);
+            const usage = late?.ok ? late.result.usage : undefined;
+            return { ok: false, error: signal.reason, ...(usage && { usage }) };
         }
         finally {
             clearTimeout(timer);
+            abort.cancel();
         }
     }
-    async executeFnTarget(evalCase) {
+    async executeFnTarget(evalCase, context) {
         const start = Date.now();
-        const output = await this.target.fn(evalCase.input);
+        const output = await this.target.fn(evalCase.input, context);
         return { case: evalCase, output, duration: Date.now() - start };
     }
-    async executeAgentTarget(evalCase) {
+    async executeAgentTarget(evalCase, signal) {
         const start = Date.now();
         const cogitator = this.target.cogitator;
         const runResult = await cogitator.run(this.target.agent, {
             input: evalCase.input,
             context: evalCase.context,
+            signal,
         });
         const duration = Date.now() - start;
         const result = {
@@ -196,8 +309,9 @@ export class EvalSuite {
             output: runResult.output ?? '',
             duration,
         };
-        if (runResult.usage) {
-            result.usage = runResult.usage;
+        const usage = targetUsageOf(runResult.usage);
+        if (usage) {
+            result.usage = usage;
         }
         if (runResult.toolCalls) {
             result.toolCalls = runResult.toolCalls;
@@ -207,7 +321,36 @@ export class EvalSuite {
     async evaluateCaseMetrics(result) {
         if (this.boundMetrics.length === 0)
             return [];
-        return Promise.all(this.boundMetrics.map((m) => m(result)));
+        return Promise.all(this.boundMetrics.map((m) => this.scoreMetric(m, result)));
+    }
+    /**
+     * A metric's score for one case. A metric that throws, or whose score is not a finite number,
+     * scores 0 with the reason in `error`, so one bad metric neither drops the case nor poisons
+     * the aggregate with NaN.
+     */
+    async scoreMetric(metric, result) {
+        try {
+            const score = await metric(result);
+            if (typeof score !== 'object' || score === null) {
+                const error = `metric returned ${String(score)} instead of a score`;
+                return { name: metric.metricName, score: 0, details: `metric error: ${error}`, error };
+            }
+            const scored = score;
+            const invalid = nonFiniteScoreError(scored.score);
+            if (!invalid)
+                return scored;
+            return {
+                ...scored,
+                name: scored.name ?? metric.metricName,
+                score: 0,
+                details: `metric error: ${invalid}`,
+                error: invalid,
+            };
+        }
+        catch (err) {
+            const error = errorMessage(err);
+            return { name: metric.metricName, score: 0, details: `metric error: ${error}`, error };
+        }
     }
     aggregateScores(results) {
         const scoresByMetric = new Map();

package/dist/eval-suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAIpE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0CrC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,MAAM,OAAO,GAAiD,EAAE,CAAC;QAEjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;wBACpD,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;wBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC1B,MAAM,EAAE,CAAC;wBAET,KAAK,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;6BAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CACnB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CACnF;6BACA,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;4BACf,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzC,SAAS,EAAE,CAAC;4BACZ,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACjE,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE;4BACZ,MAAM,EAAE,CAAC;4BACT,IAAI,OAAO,IAAI,KAAK,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gCACrC,OAAO,EAAE,CAAC;4BACZ,CAAC;iCAAM,CAAC;gCACN,KAAK,EAAE,CAAC;4BACV,CAAC;wBACH,CAAC,CAAC,CAAC;oBACP,CAAC;gBACH,CAAC,CAAC;gBACF,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAC9C,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAElE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;gBAC5B,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAkB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;SAC1E,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE;YACzB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,KAAgD,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,KAAK,GAAG,UAAU,CAChB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EAC5D,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAkB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;QAC5D,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyErC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAG3B,MAAM,cAAc,GAAG,IAAI,KAAK,CAAe,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,cAAc,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzD,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACrF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,cAAc,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBAChD,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC1D,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAA0B;QAE1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}

package/dist/index.d.ts

@@ -1,8 +1,8 @@
-export declare const VERSION = "0.1.0";
+export declare const VERSION: string;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
-export type { EvalTarget, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, } from './eval-suite.js';
+export type { EvalTarget, EvalTargetContext, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
-export type { EvalComparisonOptions, MetricComparison, ComparisonResult } from './eval-comparison.js';
+export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ComparisonResult, } from './eval-comparison.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
@@ -12,12 +12,12 @@ export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeCo
 export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
 export { latency, cost, tokenUsage } from './metrics/statistical.js';
 export { metric } from './metrics/custom.js';
-export type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
+export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
 export type { CustomMetricConfig } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
-export type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions/index.js';
+export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export type { ReporterType, ReporterOptions } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEnF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -1,4 +1,6 @@
-export const VERSION = '0.1.0';
+import { createRequire } from 'node:module';
+const packageJson = createRequire(import.meta.url)('../package.json');
+export const VERSION = packageJson.version;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
 export { EvalBuilder } from './eval-builder.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAU1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/custom.d.ts

@@ -14,5 +14,10 @@ export interface CustomMetricConfig {
         details?: string;
     };
 }
+/**
+ * A metric from an `evaluate` function. Its score is clamped to 0 - 1. When `evaluate` throws or
+ * returns a score that is not a finite number (such as NaN from a division by zero), the metric
+ * scores 0 and reports the failure in `error`.
+ */
 export declare function metric(config: CustomMetricConfig): MetricFn;
 //# sourceMappingURL=custom.d.ts.map

package/dist/metrics/custom.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,SAAS,CAAC;AAExD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CAAC,IAAI,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CA4B3D"}
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,SAAS,CAAC;AAGxD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CAAC,IAAI,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CAsC3D"}

package/dist/metrics/custom.js

@@ -1,3 +1,9 @@
+import { nonFiniteScoreError } from './types.js';
+/**
+ * A metric from an `evaluate` function. Its score is clamped to 0 - 1. When `evaluate` throws or
+ * returns a score that is not a finite number (such as NaN from a division by zero), the metric
+ * scores 0 and reports the failure in `error`.
+ */
 export function metric(config) {
     const fn = (async (result) => {
         try {
@@ -7,18 +13,28 @@ export function metric(config) {
                 expected: result.case.expected,
                 context: result.case.context,
             });
-            const clamped = Math.max(0, Math.min(1, score));
+            const invalid = nonFiniteScoreError(score);
+            if (invalid) {
+                return {
+                    name: config.name,
+                    score: 0,
+                    details: `evaluate error: ${invalid}`,
+                    error: invalid,
+                };
+            }
             return {
                 name: config.name,
-                score: clamped,
+                score: Math.max(0, Math.min(1, score)),
                 ...(details !== undefined && { details }),
             };
         }
         catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
             return {
                 name: config.name,
                 score: 0,
-                details: `evaluate error: ${err.message}`,
+                details: `evaluate error: ${message}`,
+                error: message,
             };
         }
     });

package/dist/metrics/custom.js.map

@@ -1 +1 @@
-{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAYA,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAEhD,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAoB,GAAa,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAY9C;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;gBACrC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -1,11 +1,16 @@
 import type { ZodType } from 'zod';
-import type { MetricFn } from './types.js';
-interface MatchOptions {
+import type { MetricFn, MetricOptions } from './types.js';
+interface MatchOptions extends MetricOptions {
     caseSensitive?: boolean;
 }
 export declare function exactMatch(opts?: MatchOptions): MetricFn;
 export declare function contains(opts?: MatchOptions): MetricFn;
-export declare function regex(pattern: string | RegExp): MetricFn;
-export declare function jsonSchema(schema: ZodType): MetricFn;
+/**
+ * Scores 1 when the output matches `pattern`. The `g` and `y` flags are dropped: they make
+ * `RegExp.test` resume from the previous match, so identical outputs would alternate between
+ * pass and fail.
+ */
+export declare function regex(pattern: string | RegExp, opts?: MetricOptions): MetricFn;
+export declare function jsonSchema(schema: ZodType, opts?: MetricOptions): MetricFn;
 export {};
 //# sourceMappingURL=deterministic.d.ts.map

package/dist/metrics/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,UAAU,YAAY;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAmBxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAmBtD;AAED,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAWxD;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,GAAG,QAAQ,CAwBpD"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}