@cogitator-ai/evals 0.10.0 → 0.12.0

package/dist/eval-comparison.js

@@ -1,51 +1,98 @@
 import { EvalSuite } from './eval-suite.js';
-import { EvalComparisonConfigSchema } from './schema.js';
+import { EvalComparisonConfigSchema, JudgeConfigSchema } from './schema.js';
+import { judgeContextFor } from './metrics/llm-judge.js';
+import { judgePairwise } from './pairwise.js';
+import { parseOptions } from './options.js';
 import { pairedTTest } from './stats/t-test.js';
-import { mcnemarsTest } from './stats/mcnemar.js';
+import { mcnemarExact, mcnemarsTest } from './stats/mcnemar.js';
+import { wilcoxonSignedRank } from './stats/wilcoxon.js';
+import { bootstrapCI } from './stats/bootstrap.js';
+import { cohensD } from './stats/effect-size.js';
+import { holm } from './stats/holm.js';
 import { mean } from './stats/percentiles.js';
+import { pairedDifference } from './stats/precision.js';
 function isBinary(scores) {
     return scores.every((s) => s === 0 || s === 1);
 }
-function compareMetric(baselineScores, challengerScores) {
-    const baselineMean = mean(baselineScores);
-    const challengerMean = mean(challengerScores);
-    const n = baselineScores.length;
-    if (n < 2) {
-        return {
-            baseline: baselineMean,
-            challenger: challengerMean,
-            n,
-            pValue: 1,
-            significant: false,
-            winner: 'tie',
-        };
-    }
-    const binary = isBinary(baselineScores) && isBinary(challengerScores);
-    let pValue;
-    let significant;
-    if (binary) {
-        let bCorrect_cIncorrect = 0;
-        let bIncorrect_cCorrect = 0;
-        for (let i = 0; i < n; i++) {
-            if (baselineScores[i] === 1 && challengerScores[i] === 0)
-                bCorrect_cIncorrect++;
-            if (baselineScores[i] === 0 && challengerScores[i] === 1)
-                bIncorrect_cCorrect++;
+/** Below this many discordant pairs McNemar's test is exact */
+const EXACT_MCNEMAR_BELOW = 25;
+function pairedTest(baseline, challenger, continuous, binaryAllowed) {
+    const direction = Math.sign(mean(differencesOf(baseline, challenger)));
+    if (baseline.length < 2)
+        return { test: 'none', pValue: 1, direction };
+    if (binaryAllowed && isBinary(baseline) && isBinary(challenger)) {
+        let b = 0;
+        let c = 0;
+        for (let i = 0; i < baseline.length; i++) {
+            if (baseline[i] === 1 && challenger[i] === 0)
+                b++;
+            if (baseline[i] === 0 && challenger[i] === 1)
+                c++;
         }
-        const result = mcnemarsTest(bCorrect_cIncorrect, bIncorrect_cCorrect);
-        pValue = result.pValue;
-        significant = result.significant;
+        return b + c < EXACT_MCNEMAR_BELOW
+            ? { test: 'mcnemar-exact', pValue: mcnemarExact(b, c).pValue, direction }
+            : { test: 'mcnemar', pValue: mcnemarsTest(b, c).pValue, direction };
     }
-    else {
-        const result = pairedTTest(baselineScores, challengerScores);
-        pValue = result.pValue;
-        significant = result.significant;
+    if (continuous === 'wilcoxon') {
+        const { statistic, n, pValue } = wilcoxonSignedRank(baseline, challenger);
+        return { test: 'wilcoxon', pValue, direction: Math.sign((n * (n + 1)) / 4 - statistic) };
     }
-    let winner = 'tie';
-    if (significant) {
-        winner = challengerMean > baselineMean ? 'challenger' : 'baseline';
+    return { test: 't', pValue: pairedTTest(baseline, challenger).pValue, direction };
+}
+/** Challenger minus baseline per case, without float noise */
+function differencesOf(baseline, challenger) {
+    return challenger.map((score, i) => pairedDifference(score, baseline[i]));
+}
+function meanOrNaN(values) {
+    return values.length === 0 ? Number.NaN : mean(values);
+}
+function compareMetric(name, baselineScores, challengerScores, config) {
+    const differences = differencesOf(baselineScores, challengerScores);
+    const { low, high } = bootstrapCI(differences, {
+        confidence: 1 - config.alpha,
+        ...(config.seed !== undefined && { seed: `${String(config.seed)}:${name}` }),
+    });
+    return {
+        baseline: mean(baselineScores),
+        challenger: mean(challengerScores),
+        n: baselineScores.length,
+        difference: mean(differences),
+        ci: { low, high },
+        effectSize: cohensD(differences),
+        ...pairedTest(baselineScores, challengerScores, config.test, true),
+    };
+}
+/** Each metric's verdict once its p-value is adjusted with Holm's correction over all metrics */
+function decideMetrics(compared, alpha) {
+    const names = Object.keys(compared);
+    const adjusted = holm(names.map((name) => compared[name].pValue));
+    const decided = {};
+    for (const [at, name] of names.entries()) {
+        const { direction, ...metric } = compared[name];
+        const significant = adjusted[at] < alpha && direction !== 0;
+        decided[name] = {
+            ...metric,
+            adjustedPValue: adjusted[at],
+            significant,
+            winner: !significant ? 'tie' : direction > 0 ? 'challenger' : 'baseline',
+        };
     }
-    return { baseline: baselineMean, challenger: challengerMean, n, pValue, significant, winner };
+    return decided;
+}
+function compareResource(baseline, challenger, config) {
+    const difference = meanOrNaN(differencesOf(baseline, challenger));
+    const { test, pValue, direction } = pairedTest(baseline, challenger, config.test, false);
+    const significant = pValue < config.alpha && direction !== 0;
+    return {
+        baseline: meanOrNaN(baseline),
+        challenger: meanOrNaN(challenger),
+        n: baseline.length,
+        difference,
+        test,
+        pValue,
+        significant,
+        winner: !significant ? 'tie' : direction < 0 ? 'challenger' : 'baseline',
+    };
 }
 function determineOverallWinner(metrics) {
     let baselineWins = 0;
@@ -64,14 +111,31 @@ function determineOverallWinner(metrics) {
 }
 export class EvalComparison {
     opts;
+    config;
+    pairwiseJudge;
     constructor(opts) {
-        EvalComparisonConfigSchema.parse({
+        this.config = parseOptions(EvalComparisonConfigSchema, {
             concurrency: opts.concurrency,
             timeout: opts.timeout,
             retries: opts.retries,
             trials: opts.trials,
-        });
+            alpha: opts.alpha,
+            test: opts.test,
+            seed: opts.seed,
+            pairwise: opts.pairwise,
+        }, 'EvalComparison');
         this.opts = opts;
+        if (this.config.pairwise) {
+            if (!opts.judge)
+                throw new Error('Pairwise judging requires a judge config');
+            const judgeConfig = JudgeConfigSchema.parse(opts.judge);
+            const cogitator = judgeConfig.cogitator ??
+                (opts.targets.baseline.cogitator ?? opts.targets.challenger.cogitator);
+            if (!cogitator) {
+                throw new Error('Pairwise judging needs a Cogitator to run the judge: pass judge.cogitator, or use an agent target');
+            }
+            this.pairwiseJudge = judgeContextFor(cogitator, judgeConfig);
+        }
     }
     async run(options = {}) {
         const { dataset, targets, metrics, statisticalMetrics, judge, concurrency, timeout, retries, trials, onProgress, } = this.opts;
@@ -112,21 +176,64 @@ export class EvalComparison {
                 }
             }
         }
-        const metricComparisons = {};
+        const compared = {};
         for (const name of metricNames) {
             const [baselineScores, challengerScores] = pairedScores(baselineResult, challengerResult, name);
-            metricComparisons[name] = compareMetric(baselineScores, challengerScores);
+            compared[name] = compareMetric(name, baselineScores, challengerScores, this.config);
         }
+        const metricComparisons = decideMetrics(compared, this.config.alpha);
+        const latency = compareResource(...pairedResource(baselineResult, challengerResult, caseLatency), this.config);
+        const [baselineCost, challengerCost] = pairedResource(baselineResult, challengerResult, caseCost);
+        const cost = baselineCost.length > 0
+            ? compareResource(baselineCost, challengerCost, this.config)
+            : undefined;
+        const pairwise = this.pairwiseJudge && this.config.pairwise
+            ? await judgePairwise(baselineResult, challengerResult, this.pairwiseJudge, this.config.pairwise, this.config.concurrency, options.signal)
+            : undefined;
         return {
             summary: {
                 winner: determineOverallWinner(metricComparisons),
                 metrics: metricComparisons,
+                latency,
+                ...(cost && { cost }),
             },
             baseline: baselineResult,
             challenger: challengerResult,
+            ...(pairwise && { pairwise }),
         };
     }
 }
+/** The trials of a case that answered */
+function answeredTrials(result) {
+    return (result.trials ?? [result]).filter((trial) => trial.error === undefined);
+}
+/** A case's latency: the mean duration of its answered trials, none when none answered */
+function caseLatency(result) {
+    const answered = answeredTrials(result);
+    return answered.length === 0 ? undefined : mean(answered.map((trial) => trial.duration));
+}
+/** A case's cost: the mean cost of its answered trials that reported usage */
+function caseCost(result) {
+    const costs = answeredTrials(result).flatMap((trial) => trial.usage === undefined ? [] : [trial.usage.cost]);
+    return costs.length === 0 ? undefined : mean(costs);
+}
+/** A per-case figure of both sides, over the cases both have it for */
+function pairedResource(baseline, challenger, figure) {
+    const baselineValues = [];
+    const challengerValues = [];
+    for (let at = 0; at < baseline.results.length; at++) {
+        const challengerResult = challenger.results[at];
+        if (!challengerResult)
+            continue;
+        const b = figure(baseline.results[at]);
+        const c = figure(challengerResult);
+        if (b === undefined || c === undefined)
+            continue;
+        baselineValues.push(b);
+        challengerValues.push(c);
+    }
+    return [baselineValues, challengerValues];
+}
 /** A case's score of a metric: the mean over its trials the metric scored, none when it scored none */
 function caseScore(result, metricName) {
     if (!result)

package/dist/eval-comparison.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AA8C3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,CAAC;YACD,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAChG,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAgHrD,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,+DAA+D;AAC/D,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAY/B,SAAS,UAAU,CACjB,QAAkB,EAClB,UAAoB,EACpB,UAA4B,EAC5B,aAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;IACvE,IAAI,aAAa,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;YAClD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,mBAAmB;YAChC,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE;YACzE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3F,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;AACpF,CAAC;AAED,8DAA8D;AAC9D,SAAS,aAAa,CAAC,QAAkB,EAAE,UAAoB;IAC7D,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAMD,SAAS,SAAS,CAAC,MAAgB;IACjC,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,cAAwB,EACxB,gBAA0B,EAC1B,MAA4B;IAE5B,MAAM,WAAW,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,WAAW,EAAE;QAC7C,UAAU,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK;QAC5B,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC;QAC9B,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC;QAClC,CAAC,EAAE,cAAc,CAAC,MAAM;QACxB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC;QAC7B,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;QACjB,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;QAChC,GAAG,UAAU,CAAC,cAAc,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,iGAAiG;AACjG,SAAS,aAAa,CACpB,QAAwC,EACxC,KAAa;IAEb,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,GAAG;YACd,GAAG,MAAM;YACT,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC5B,WAAW;YACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;SACzE,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CACtB,QAAkB,EAClB,UAAoB,EACpB,MAA4B;IAE5B,MAAM,UAAU,GAAG,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;IAC7D,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC7B,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC;QACjC,CAAC,EAAE,QAAQ,CAAC,MAAM;QAClB,UAAU;QACV,IAAI;QACJ,MAAM;QACN,WAAW;QACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;KACzE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAyC;IACvE,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAC5B,MAAM,CAAuB;IAC7B,aAAa,CAAgB;IAE9C,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,YAAY,CACxB,0BAA0B,EAC1B;YACE,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,gBAAgB,CACjB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,SAAS,GACb,WAAW,CAAC,SAAS;gBACpB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CACzC,CAAC;YAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAmC,EAAE,CAAC;QACpD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,cAAc,EAAE,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,iBAAiB,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAErE,MAAM,OAAO,GAAG,eAAe,CAC7B,GAAG,cAAc,CAAC,cAAc,EAAE,gBAAgB,EAAE,WAAW,CAAC,EAChE,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,cAAc,CACnD,cAAc,EACd,gBAAgB,EAChB,QAAQ,CACT,CAAC;QACF,MAAM,IAAI,GACR,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC;YAC5D,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,QAAQ,GACZ,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;YACxC,CAAC,CAAC,MAAM,aAAa,CACjB,cAAc,EACd,gBAAgB,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CAAC,WAAW,EACvB,OAAO,CAAC,MAAM,CACf;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;gBAC1B,OAAO;gBACP,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;aACtB;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;YAC5B,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;CACF;AAED,yCAAyC;AACzC,SAAS,cAAc,CAAC,MAAwB;IAC9C,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;AAClF,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,MAAwB;IACxC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACpD,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,uEAAuE;AACvE,SAAS,cAAc,CACrB,QAAyB,EACzB,UAA2B,EAC3B,MAAwD;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}

package/dist/index.d.ts

@@ -2,14 +2,19 @@ export declare const VERSION: string;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
 export type { EvalTarget, EvalTargetContext, EvalTurn, RetrievalToolResult, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
-export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ComparisonResult, } from './eval-comparison.js';
+export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ResourceComparison, ComparisonResult, ComparisonTest, ComparisonWinner, } from './eval-comparison.js';
+export type { PairwiseOptions, PairwiseResult, PairwiseCase, PairwiseVerdict, PairwiseSide, PairwiseOutcome, } from './pairwise.js';
+export { calibrateJudge } from './calibrate-judge.js';
+export type { CalibrationExample, CalibrateJudgeOptions, CalibrationCase, JudgeCalibration, } from './calibrate-judge.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
-export type { LLMMetricFn, JudgeContext, JudgeMessage } from './metrics/llm-judge.js';
+export type { LLMMetricFn, LLMMetricOptions, PromptMetricOptions, RubricMetricOptions, RubricLevel, JudgeContext, JudgeMessage, JudgeRunOptions, JudgeFormat, JudgeVerdict, JudgeScore, JudgeSetup, } from './metrics/llm-judge.js';
+export { gEval } from './metrics/g-eval.js';
+export type { GEvalOptions } from './metrics/g-eval.js';
 export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
 export type { PassAtKOptions, PassAtKMetricFn } from './metrics/statistical.js';
 export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
@@ -28,8 +33,15 @@ export { report } from './reporters/index.js';
 export type { ReporterType, ReporterOptions, ReportedCase } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export type { TTestResult } from './stats/t-test.js';
-export { mcnemarsTest } from './stats/mcnemar.js';
-export type { McNemarResult } from './stats/mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';
+export type { McNemarResult, McNemarExactResult } from './stats/mcnemar.js';
+export { wilcoxonSignedRank } from './stats/wilcoxon.js';
+export type { WilcoxonResult } from './stats/wilcoxon.js';
+export { bootstrapCI } from './stats/bootstrap.js';
+export type { BootstrapOptions, BootstrapInterval } from './stats/bootstrap.js';
+export { cohensD } from './stats/effect-size.js';
+export { holm } from './stats/holm.js';
+export { pearson, spearman } from './stats/correlation.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
 export type { EvalCase, ExpectedToolCall, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
 export { createRunEvalTool, evalTools } from './tools.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,eAAe,EACf,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,YAAY,EACZ,UAAU,EACV,UAAU,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -3,12 +3,14 @@ const packageJson = createRequire(import.meta.url)('../package.json');
 export const VERSION = packageJson.version;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
+export { calibrateJudge } from './calibrate-judge.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export { gEval } from './metrics/g-eval.js';
 export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
 export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
 export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
@@ -20,7 +22,12 @@ export { assertion } from './assertions/index.js';
 export { maxErrorRate } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';
-export { mcnemarsTest } from './stats/mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';
+export { wilcoxonSignedRank } from './stats/wilcoxon.js';
+export { bootstrapCI } from './stats/bootstrap.js';
+export { cohensD } from './stats/effect-size.js';
+export { holm } from './stats/holm.js';
+export { pearson, spearman } from './stats/correlation.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
 export { createRunEvalTool, evalTools } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAkBnD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAe7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -5,6 +5,7 @@ interface MatchOptions extends MetricOptions {
 }
 export declare function exactMatch(opts?: MatchOptions): MetricFn;
 export declare function similarity(opts?: MatchOptions): MetricFn;
+export declare function tokenF1(opts?: MatchOptions): MetricFn;
 export declare function contains(opts?: MatchOptions): MetricFn;
 /** Scores the fraction of keywords found in the output. */
 export declare function containsAll(keywords: string[], opts?: MatchOptions): MetricFn;

package/dist/metrics/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAkCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AA8BD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAaD,wBAAgB,OAAO,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CA+CrD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}

package/dist/metrics/deterministic.js

@@ -55,10 +55,7 @@ function levenshteinDistance(s1, s2) {
         const char1 = s1[i - 1];
         for (let j = 1; j <= n; j++) {
             const cost = char1 === s2[j - 1] ? 0 : 1;
-            curr[j] = Math.min(prev[j] + 1, // deletion
-            curr[j - 1] + 1, // insertion
-            prev[j - 1] + cost // substitution
-            );
+            curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
         }
         const temp = prev;
         prev = curr;
@@ -93,6 +90,54 @@ export function similarity(opts) {
         };
     }));
 }
+function tokenize(text, caseSensitive) {
+    const normalized = (caseSensitive ? text : text.toLowerCase())
+        .replace(/[!-/:-@[-`{-~]/g, '')
+        .replace(/\b(a|an|the)\b/gi, ' ');
+    return normalized.split(/\s+/).filter(Boolean);
+}
+function roundToTwo(value) {
+    return Number(value.toFixed(2));
+}
+export function tokenF1(opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'tokenF1';
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expected;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected value provided' };
+        }
+        const output = result.output.trim();
+        const target = expected.trim();
+        const actualTokens = tokenize(output, caseSensitive);
+        const expectedTokens = tokenize(target, caseSensitive);
+        if (actualTokens.length === 0 && expectedTokens.length === 0) {
+            return { name, score: 1 };
+        }
+        const counts = new Map();
+        for (const token of expectedTokens) {
+            counts.set(token, (counts.get(token) ?? 0) + 1);
+        }
+        let overlap = 0;
+        for (const token of actualTokens) {
+            const left = counts.get(token) ?? 0;
+            if (left > 0) {
+                overlap++;
+                counts.set(token, left - 1);
+            }
+        }
+        const precision = actualTokens.length > 0 ? overlap / actualTokens.length : 0;
+        const recall = expectedTokens.length > 0 ? overlap / expectedTokens.length : 0;
+        const score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+        return {
+            name,
+            score,
+            details: score < 1
+                ? `expected "${target}", got "${output}" (precision: ${roundToTwo(precision)}, recall: ${roundToTwo(recall)})`
+                : undefined,
+        };
+    }));
+}
 export function contains(opts) {
     const caseSensitive = opts?.caseSensitive ?? false;
     const name = opts?.name ?? 'contains';

package/dist/metrics/deterministic.js.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY;YAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,aAAsB;IACpD,MAAM,UAAU,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;SAC3D,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,SAAS,CAAC;IAErC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEvD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3F,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC;gBACP,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,iBAAiB,UAAU,CAAC,SAAS,CAAC,aAAa,UAAU,CAAC,MAAM,CAAC,GAAG;gBAC9G,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/g-eval.d.ts

@@ -0,0 +1,29 @@
+import type { LLMMetricFn } from './llm-judge.js';
+export interface GEvalOptions {
+    name: string;
+    /** What a good response does, in your words */
+    criteria: string;
+    /** The evaluation steps the judge follows. Default: the judge writes them from `criteria` */
+    steps?: readonly string[];
+    /** The scale the judge scores on, `[min, max]`, normalized to 0 - 1. Default `[1, 5]` */
+    scale?: readonly [number, number];
+}
+/**
+ * The steps of a numbered or bulleted list, in order: `1. ...`, `1) ...`, `Step 1: ...`,
+ * `- ...`, a Markdown bold marker (`**1.** ...`, `**Step 1:** ...`) included. A bullet needs a
+ * space after it, so a rule (`---`) or emphasis (`*note*`) is no step, and the text of a step is
+ * kept as written
+ */
+export declare function stepsOf(text: string): string[];
+/**
+ * G-Eval: the judge scores each case on `scale` following evaluation steps, the score normalized
+ * to 0 - 1. Without `steps`, the judge writes them from `criteria` once per metric instance,
+ * before the first case it scores, and every later case reuses them. A failed generation scores
+ * the cases waiting on it as judge errors, and the next case tries again.
+ *
+ * The original G-Eval weights each score of the scale by the probability of its token, which not
+ * every provider exposes. Set `judge.samples` (with a `temperature` above 0) to average several
+ * verdicts instead.
+ */
+export declare function gEval(opts: GEvalOptions): LLMMetricFn;
+//# sourceMappingURL=g-eval.d.ts.map

package/dist/metrics/g-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"g-eval.d.ts","sourceRoot":"","sources":["../../src/metrics/g-eval.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAsD,WAAW,EAAE,MAAM,aAAa,CAAC;AAEnG,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,6FAA6F;IAC7F,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,yFAAyF;IACzF,KAAK,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAkBD;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAK9C;AAaD;;;;;;;;;GASG;AACH,wBAAgB,KAAK,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,CAiDrD"}

package/dist/metrics/g-eval.js

@@ -0,0 +1,89 @@
+import { z } from 'zod';
+import { parseOptions } from '../options.js';
+import { JudgeUsageError, createJudgeMetric, usageOf } from './llm-judge.js';
+const GEvalOptionsSchema = z.object({
+    name: z.string().trim().min(1, 'name must not be empty'),
+    criteria: z.string().trim().min(1, 'criteria must not be empty'),
+    steps: z
+        .array(z.string().trim().min(1, 'a step must not be empty'))
+        .min(1, 'give at least one step, or leave steps out for the judge to write them')
+        .optional(),
+    scale: z
+        .tuple([z.number(), z.number()])
+        .refine(([min, max]) => min < max, 'scale must be [min, max] with min below max')
+        .default([1, 5]),
+});
+const STEP_LINE = /^\s*(?:(\*\*|__)?(?:step\s*)?\d+\s*[.):-]\1?\s*|(?:\*\*|__)?[-*\u2022](?:\*\*|__)?\s+)(.*\S)\s*$/i;
+/**
+ * The steps of a numbered or bulleted list, in order: `1. ...`, `1) ...`, `Step 1: ...`,
+ * `- ...`, a Markdown bold marker (`**1.** ...`, `**Step 1:** ...`) included. A bullet needs a
+ * space after it, so a rule (`---`) or emphasis (`*note*`) is no step, and the text of a step is
+ * kept as written
+ */
+export function stepsOf(text) {
+    return text
+        .split('\n')
+        .map((line) => STEP_LINE.exec(line)?.[2])
+        .filter((step) => step !== undefined);
+}
+const hasSteps = (output) => stepsOf(output).length > 0;
+function stepsRequest(criteria) {
+    return `Criteria for evaluating an assistant's response to an input:\n${criteria}\n\nWrite the evaluation steps a careful grader follows to score a response on these criteria, 3 to 6 short steps. Reply with a numbered list, one step per line, and nothing else.`;
+}
+function scoringPrompt(criteria, steps, [min, max]) {
+    const numbered = steps.map((step, i) => `${i + 1}. ${step}`).join('\n');
+    return `You are evaluating an assistant's response on the criteria below.\n\nCriteria:\n${criteria}\n\nEvaluation steps:\n${numbered}\n\nFollow the evaluation steps, then score the <response> from ${min} (does not meet the criteria at all) to ${max} (fully meets them).\n\nRespond with JSON: {"score": <number from ${min} to ${max}>, "reasoning": "<explanation>"}`;
+}
+/**
+ * G-Eval: the judge scores each case on `scale` following evaluation steps, the score normalized
+ * to 0 - 1. Without `steps`, the judge writes them from `criteria` once per metric instance,
+ * before the first case it scores, and every later case reuses them. A failed generation scores
+ * the cases waiting on it as judge errors, and the next case tries again.
+ *
+ * The original G-Eval weights each score of the scale by the probability of its token, which not
+ * every provider exposes. Set `judge.samples` (with a `temperature` above 0) to average several
+ * verdicts instead.
+ */
+export function gEval(opts) {
+    const { name, criteria, steps, scale } = parseOptions(GEvalOptionsSchema, opts, 'gEval');
+    const [min, max] = scale;
+    let generating;
+    const generate = async (context) => {
+        const answered = await context.cogitator.run({
+            input: stepsRequest(criteria),
+            format: 'text',
+            accept: hasSteps,
+        });
+        const usage = usageOf(answered.usage);
+        const written = stepsOf(answered.output);
+        if (written.length === 0) {
+            throw new JudgeUsageError('could not read evaluation steps from the judge, it gave no numbered list', usage);
+        }
+        return { systemPrompt: scoringPrompt(criteria, written, scale), ...(usage && { usage }) };
+    };
+    const setup = async (context) => {
+        if (steps)
+            return { systemPrompt: scoringPrompt(criteria, steps, scale) };
+        if (generating) {
+            try {
+                return { systemPrompt: (await generating).systemPrompt };
+            }
+            catch (error) {
+                throw new Error(error instanceof Error ? error.message : String(error), { cause: error });
+            }
+        }
+        const started = generate(context).catch((error) => {
+            if (generating === started)
+                generating = undefined;
+            const message = error instanceof Error ? error.message : String(error);
+            throw new JudgeUsageError(`gEval could not generate its evaluation steps: ${message}`, error instanceof JudgeUsageError ? error.usage : undefined, { cause: error });
+        });
+        generating = started;
+        return started;
+    };
+    const score = ({ score: raw }) => ({
+        score: (Math.max(min, Math.min(max, raw)) - min) / (max - min),
+    });
+    return createJudgeMetric(name, scoringPrompt(criteria, steps ?? [], scale), { setup, score });
+}
+//# sourceMappingURL=g-eval.js.map