@cogitator-ai/evals 0.12.0 → 0.13.0

package/dist/eval-comparison.js

@@ -3,112 +3,8 @@ import { EvalComparisonConfigSchema, JudgeConfigSchema } from './schema.js';
 import { judgeContextFor } from './metrics/llm-judge.js';
 import { judgePairwise } from './pairwise.js';
 import { parseOptions } from './options.js';
-import { pairedTTest } from './stats/t-test.js';
-import { mcnemarExact, mcnemarsTest } from './stats/mcnemar.js';
-import { wilcoxonSignedRank } from './stats/wilcoxon.js';
-import { bootstrapCI } from './stats/bootstrap.js';
-import { cohensD } from './stats/effect-size.js';
-import { holm } from './stats/holm.js';
 import { mean } from './stats/percentiles.js';
-import { pairedDifference } from './stats/precision.js';
-function isBinary(scores) {
-    return scores.every((s) => s === 0 || s === 1);
-}
-/** Below this many discordant pairs McNemar's test is exact */
-const EXACT_MCNEMAR_BELOW = 25;
-function pairedTest(baseline, challenger, continuous, binaryAllowed) {
-    const direction = Math.sign(mean(differencesOf(baseline, challenger)));
-    if (baseline.length < 2)
-        return { test: 'none', pValue: 1, direction };
-    if (binaryAllowed && isBinary(baseline) && isBinary(challenger)) {
-        let b = 0;
-        let c = 0;
-        for (let i = 0; i < baseline.length; i++) {
-            if (baseline[i] === 1 && challenger[i] === 0)
-                b++;
-            if (baseline[i] === 0 && challenger[i] === 1)
-                c++;
-        }
-        return b + c < EXACT_MCNEMAR_BELOW
-            ? { test: 'mcnemar-exact', pValue: mcnemarExact(b, c).pValue, direction }
-            : { test: 'mcnemar', pValue: mcnemarsTest(b, c).pValue, direction };
-    }
-    if (continuous === 'wilcoxon') {
-        const { statistic, n, pValue } = wilcoxonSignedRank(baseline, challenger);
-        return { test: 'wilcoxon', pValue, direction: Math.sign((n * (n + 1)) / 4 - statistic) };
-    }
-    return { test: 't', pValue: pairedTTest(baseline, challenger).pValue, direction };
-}
-/** Challenger minus baseline per case, without float noise */
-function differencesOf(baseline, challenger) {
-    return challenger.map((score, i) => pairedDifference(score, baseline[i]));
-}
-function meanOrNaN(values) {
-    return values.length === 0 ? Number.NaN : mean(values);
-}
-function compareMetric(name, baselineScores, challengerScores, config) {
-    const differences = differencesOf(baselineScores, challengerScores);
-    const { low, high } = bootstrapCI(differences, {
-        confidence: 1 - config.alpha,
-        ...(config.seed !== undefined && { seed: `${String(config.seed)}:${name}` }),
-    });
-    return {
-        baseline: mean(baselineScores),
-        challenger: mean(challengerScores),
-        n: baselineScores.length,
-        difference: mean(differences),
-        ci: { low, high },
-        effectSize: cohensD(differences),
-        ...pairedTest(baselineScores, challengerScores, config.test, true),
-    };
-}
-/** Each metric's verdict once its p-value is adjusted with Holm's correction over all metrics */
-function decideMetrics(compared, alpha) {
-    const names = Object.keys(compared);
-    const adjusted = holm(names.map((name) => compared[name].pValue));
-    const decided = {};
-    for (const [at, name] of names.entries()) {
-        const { direction, ...metric } = compared[name];
-        const significant = adjusted[at] < alpha && direction !== 0;
-        decided[name] = {
-            ...metric,
-            adjustedPValue: adjusted[at],
-            significant,
-            winner: !significant ? 'tie' : direction > 0 ? 'challenger' : 'baseline',
-        };
-    }
-    return decided;
-}
-function compareResource(baseline, challenger, config) {
-    const difference = meanOrNaN(differencesOf(baseline, challenger));
-    const { test, pValue, direction } = pairedTest(baseline, challenger, config.test, false);
-    const significant = pValue < config.alpha && direction !== 0;
-    return {
-        baseline: meanOrNaN(baseline),
-        challenger: meanOrNaN(challenger),
-        n: baseline.length,
-        difference,
-        test,
-        pValue,
-        significant,
-        winner: !significant ? 'tie' : direction < 0 ? 'challenger' : 'baseline',
-    };
-}
-function determineOverallWinner(metrics) {
-    let baselineWins = 0;
-    let challengerWins = 0;
-    for (const mc of Object.values(metrics)) {
-        if (mc.winner === 'baseline')
-            baselineWins++;
-        if (mc.winner === 'challenger')
-            challengerWins++;
-    }
-    if (challengerWins > baselineWins)
-        return 'challenger';
-    if (baselineWins > challengerWins)
-        return 'baseline';
-    return 'tie';
-}
+import { compareMetrics, compareResource, determineOverallWinner, pairedScores, scoredMetricNames, } from './comparison-stats.js';
 export class EvalComparison {
     opts;
     config;
@@ -167,21 +63,15 @@ export class EvalComparison {
             baselineSuite.run({ signal: options.signal }),
             challengerSuite.run({ signal: options.signal }),
         ]);
-        const metricNames = new Set();
-        for (const result of [baselineResult, challengerResult]) {
-            for (const r of result.results) {
-                for (const trial of r.trials ?? [r]) {
-                    for (const s of trial.scores)
-                        metricNames.add(s.name);
-                }
-            }
+        const pairs = casePairs(baselineResult, challengerResult);
+        const paired = new Map();
+        for (const name of scoredMetricNames([
+            ...baselineResult.results,
+            ...challengerResult.results,
+        ])) {
+            paired.set(name, pairedScores(pairs, name));
         }
-        const compared = {};
-        for (const name of metricNames) {
-            const [baselineScores, challengerScores] = pairedScores(baselineResult, challengerResult, name);
-            compared[name] = compareMetric(name, baselineScores, challengerScores, this.config);
-        }
-        const metricComparisons = decideMetrics(compared, this.config.alpha);
+        const metricComparisons = compareMetrics(paired, this.config);
         const latency = compareResource(...pairedResource(baselineResult, challengerResult, caseLatency), this.config);
         const [baselineCost, challengerCost] = pairedResource(baselineResult, challengerResult, caseCost);
         const cost = baselineCost.length > 0
@@ -234,35 +124,8 @@ function pairedResource(baseline, challenger, figure) {
     }
     return [baselineValues, challengerValues];
 }
-/** A case's score of a metric: the mean over its trials the metric scored, none when it scored none */
-function caseScore(result, metricName) {
-    if (!result)
-        return undefined;
-    const trials = result.trials ?? [result];
-    if (trials.every((trial) => trial.error !== undefined))
-        return undefined;
-    const scores = trials.flatMap((trial) => trial.error !== undefined
-        ? [0]
-        : trial.scores.filter((s) => s.name === metricName).map((s) => s.score));
-    return scores.length === 0 ? undefined : mean(scores);
-}
-/**
- * The scores of a metric for each case, side by side. A case one side errored on counts as a 0
- * for that side, since a target that fails loses the case, and a case both sides errored on is
- * left out, telling nothing about either
- */
-function pairedScores(baseline, challenger, metricName) {
-    const scoreOf = (result, at) => caseScore(result.results[at], metricName);
-    const baselineScores = [];
-    const challengerScores = [];
-    for (let at = 0; at < baseline.results.length; at++) {
-        const b = scoreOf(baseline, at);
-        const c = scoreOf(challenger, at);
-        if (b === undefined && c === undefined)
-            continue;
-        baselineScores.push(b ?? 0);
-        challengerScores.push(c ?? 0);
-    }
-    return [baselineScores, challengerScores];
+/** The cases of the two runs side by side, by position */
+function casePairs(baseline, challenger) {
+    return baseline.results.map((result, at) => [result, challenger.results[at]]);
 }
 //# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAgHrD,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,+DAA+D;AAC/D,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAY/B,SAAS,UAAU,CACjB,QAAkB,EAClB,UAAoB,EACpB,UAA4B,EAC5B,aAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;IACvE,IAAI,aAAa,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;YAClD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,mBAAmB;YAChC,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE;YACzE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3F,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;AACpF,CAAC;AAED,8DAA8D;AAC9D,SAAS,aAAa,CAAC,QAAkB,EAAE,UAAoB;IAC7D,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAMD,SAAS,SAAS,CAAC,MAAgB;IACjC,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,cAAwB,EACxB,gBAA0B,EAC1B,MAA4B;IAE5B,MAAM,WAAW,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,WAAW,EAAE;QAC7C,UAAU,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK;QAC5B,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC;QAC9B,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC;QAClC,CAAC,EAAE,cAAc,CAAC,MAAM;QACxB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC;QAC7B,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;QACjB,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;QAChC,GAAG,UAAU,CAAC,cAAc,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,iGAAiG;AACjG,SAAS,aAAa,CACpB,QAAwC,EACxC,KAAa;IAEb,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,GAAG;YACd,GAAG,MAAM;YACT,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC5B,WAAW;YACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;SACzE,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CACtB,QAAkB,EAClB,UAAoB,EACpB,MAA4B;IAE5B,MAAM,UAAU,GAAG,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;IAC7D,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC7B,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC;QACjC,CAAC,EAAE,QAAQ,CAAC,MAAM;QAClB,UAAU;QACV,IAAI;QACJ,MAAM;QACN,WAAW;QACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;KACzE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAyC;IACvE,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAC5B,MAAM,CAAuB;IAC7B,aAAa,CAAgB;IAE9C,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,YAAY,CACxB,0BAA0B,EAC1B;YACE,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,gBAAgB,CACjB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,SAAS,GACb,WAAW,CAAC,SAAS;gBACpB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CACzC,CAAC;YAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAmC,EAAE,CAAC;QACpD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,cAAc,EAAE,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,iBAAiB,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAErE,MAAM,OAAO,GAAG,eAAe,CAC7B,GAAG,cAAc,CAAC,cAAc,EAAE,gBAAgB,EAAE,WAAW,CAAC,EAChE,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,cAAc,CACnD,cAAc,EACd,gBAAgB,EAChB,QAAQ,CACT,CAAC;QACF,MAAM,IAAI,GACR,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC;YAC5D,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,QAAQ,GACZ,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;YACxC,CAAC,CAAC,MAAM,aAAa,CACjB,cAAc,EACd,gBAAgB,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CAAC,WAAW,EACvB,OAAO,CAAC,MAAM,CACf;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;gBAC1B,OAAO;gBACP,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;aACtB;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;YAC5B,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;CACF;AAED,yCAAyC;AACzC,SAAS,cAAc,CAAC,MAAwB;IAC9C,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;AAClF,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,MAAwB;IACxC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACpD,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,uEAAuE;AACvE,SAAS,cAAc,CACrB,QAAyB,EACzB,UAA2B,EAC3B,MAAwD;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAyE5B,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAC5B,MAAM,CAAuB;IAC7B,aAAa,CAAgB;IAE9C,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,YAAY,CACxB,0BAA0B,EAC1B;YACE,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,gBAAgB,CACjB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,SAAS,GACb,WAAW,CAAC,SAAS;gBACpB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CACzC,CAAC;YAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc,CAAC,OAAO;YACzB,GAAG,gBAAgB,CAAC,OAAO;SAC5B,CAAC,EAAE,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAG,eAAe,CAC7B,GAAG,cAAc,CAAC,cAAc,EAAE,gBAAgB,EAAE,WAAW,CAAC,EAChE,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,cAAc,CACnD,cAAc,EACd,gBAAgB,EAChB,QAAQ,CACT,CAAC;QACF,MAAM,IAAI,GACR,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC;YAC5D,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,QAAQ,GACZ,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;YACxC,CAAC,CAAC,MAAM,aAAa,CACjB,cAAc,EACd,gBAAgB,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CAAC,WAAW,EACvB,OAAO,CAAC,MAAM,CACf;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;gBAC1B,OAAO;gBACP,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;aACtB;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;YAC5B,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;CACF;AAED,yCAAyC;AACzC,SAAS,cAAc,CAAC,MAAwB;IAC9C,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;AAClF,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,MAAwB;IACxC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACpD,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,uEAAuE;AACvE,SAAS,cAAc,CACrB,QAAyB,EACzB,UAA2B,EAC3B,MAAwD;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC;AAED,0DAA0D;AAC1D,SAAS,SAAS,CAAC,QAAyB,EAAE,UAA2B;IACvE,OAAO,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC;AACzF,CAAC"}

package/dist/eval-suite.d.ts

@@ -5,6 +5,7 @@ import type { MetricFn, Passage, ScoredCaseResult, StatisticalMetricFn } from '.
 import type { LLMMetricFn } from './metrics/llm-judge.js';
 import type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 import type { ReporterType, ReporterOptions } from './reporters/index.js';
+import type { EvalRun, JsonMetadata } from './runs/schema.js';
 /** A turn of a conversation: what the user said and what the target answered */
 export interface EvalTurn {
     input: string;
@@ -52,6 +53,13 @@ export interface EvalProgress {
 export interface EvalSuiteOptions {
     dataset: Dataset;
     target: EvalTarget;
+    /** Names the experiment the suite runs, such as `support-agent`, in reports and run files */
+    name?: string;
+    /**
+     * Describes the experiment in run files, such as `{ branch: 'main', commit: 'a1b2c3' }`. Must be
+     * JSON-serializable: keys holding `undefined` are dropped, other values JSON cannot hold throw
+     */
+    metadata?: Record<string, unknown>;
     metrics?: MetricFn[];
     statisticalMetrics?: StatisticalMetricFn[];
     judge?: JudgeConfigInput;
@@ -97,6 +105,28 @@ export interface EvalSuiteResult {
     report: (type: ReporterType | ReporterType[], options?: ReporterOptions) => void;
     /** Writes the mean of each metric as JSON; throws when a mean is not a finite number */
     saveBaseline: (path: string) => void;
+    /** Unique id of the run, which names its run file: `20261022T101530123Z-1a2b3c4d` */
+    runId: string;
+    /** The suite's `name` */
+    name?: string;
+    /** The suite's `metadata` */
+    metadata?: JsonMetadata;
+    /** When the run started, ISO 8601 */
+    startedAt: string;
+    /** When the run finished, ISO 8601 */
+    finishedAt: string;
+    /**
+     * The run as a run file holds it: id, times, name and metadata, the dataset fingerprint, the
+     * suite's configuration (never its functions, Cogitator or keys), the prompts used, every case
+     * result with its `caseKey`, the aggregates, assertions and stats
+     */
+    toRun: () => EvalRun;
+    /**
+     * Writes the run file and returns its path: `<target>/<runId>.json` when `target` is a
+     * directory or ends with a path separator, `target` itself otherwise. Read it back with
+     * `loadRun()`, keep runs with a `RunStore` and compare them with `compareRuns()`
+     */
+    saveRun: (target: string) => string;
 }
 export declare function isLLMMetric(m: MetricFn): m is LLMMetricFn;
 export declare class EvalSuite {
@@ -110,6 +140,9 @@ export declare class EvalSuite {
     private readonly retries;
     private readonly trials;
     private readonly onProgress?;
+    private readonly name?;
+    private readonly metadata?;
+    private readonly runConfig;
     constructor(opts: EvalSuiteOptions);
     private validateTarget;
     /** A `passAtK()` metric needs the metric it reads and at least `k` trials */

package/dist/eval-suite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAMR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE/E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAyDD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA6ClC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAyGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YA4ElB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAOR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG/E,OAAO,KAAK,EAAE,OAAO,EAAiB,YAAY,EAAE,MAAM,eAAe,CAAC;AAG1E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC;IACrB;;;;OAIG;IACH,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACrC;AA2DD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;IAC/D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;gBAE9B,IAAI,EAAE,gBAAgB;IAoElC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAqIjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YAgFlB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}

package/dist/eval-suite.js

@@ -7,6 +7,9 @@ import { ragSearchPassages } from './metrics/retrieval.js';
 import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
 import { aggregate, mean } from './stats/index.js';
 import { report } from './reporters/index.js';
+import { parseOptions } from './options.js';
+import { ExperimentOptionsSchema, RunPromptSchema } from './runs/schema.js';
+import { buildRun, newRunId, writeRun } from './runs/run.js';
 /**
  * A turn of a multi-turn case failed: the error of the turn, with the turns run so far (the
  * failed one last) and what they spent
@@ -118,6 +121,9 @@ export class EvalSuite {
     retries;
     trials;
     onProgress;
+    name;
+    metadata;
+    runConfig;
     constructor(opts) {
         const config = EvalSuiteConfigSchema.parse({
             concurrency: opts.concurrency,
@@ -134,6 +140,9 @@ export class EvalSuite {
         this.retries = config.retries;
         this.trials = config.trials;
         this.onProgress = opts.onProgress;
+        const experiment = parseOptions(ExperimentOptionsSchema, { name: opts.name, metadata: opts.metadata }, 'EvalSuite');
+        this.name = experiment.name;
+        this.metadata = experiment.metadata;
         this.validateTarget();
         const rawMetrics = opts.metrics ?? [];
         this.validateMetricNames(rawMetrics);
@@ -149,6 +158,20 @@ export class EvalSuite {
         }
         const judge = judgeConfig && judgeCogitator ? judgeContextFor(judgeCogitator, judgeConfig) : null;
         this.boundMetrics = rawMetrics.map((m) => isLLMMetric(m) && judge ? bindJudgeContext(m, judge) : m);
+        this.runConfig = {
+            metrics: rawMetrics.map((m) => m.metricName),
+            statisticalMetrics: this.statisticalMetrics.map((m) => m.metricName),
+            ...(judgeConfig && {
+                judge: {
+                    model: judgeConfig.model,
+                    temperature: judgeConfig.temperature,
+                    ...(judgeConfig.samples !== undefined && { samples: judgeConfig.samples }),
+                },
+            }),
+            trials: this.trials,
+            concurrency: this.concurrency,
+            target: targetDescription(this.target),
+        };
     }
     validateTarget() {
         const { fn, agent, cogitator } = this.target;
@@ -209,7 +232,9 @@ export class EvalSuite {
             throw new Error(`maxCases must be a positive integer, got ${maxCases}`);
         }
         signal?.throwIfAborted();
-        const suiteStart = Date.now();
+        const startedAt = new Date();
+        const runId = newRunId(startedAt);
+        const suiteStart = startedAt.getTime();
         const cases = this.cases(maxCases);
         const total = cases.length;
         const trials = this.trials;
@@ -255,13 +280,37 @@ export class EvalSuite {
             judgeCost,
         };
         const assertionResults = this.assertionFns.map((fn) => fn(aggregated, stats));
+        const finishedAt = new Date(suiteStart + stats.duration);
+        let run;
+        const toRun = () => (run ??= buildRun({
+            runId,
+            startedAt,
+            finishedAt,
+            ...(this.name !== undefined && { name: this.name }),
+            ...(this.metadata !== undefined && { metadata: this.metadata }),
+            cases,
+            config: this.runConfig,
+            results: orderedResults,
+            aggregated,
+            assertions: assertionResults,
+            stats,
+        }));
         const suiteResult = {
             results: orderedResults,
             aggregated,
             assertions: assertionResults,
             stats,
+            runId,
+            ...(this.name !== undefined && { name: this.name }),
+            ...(this.metadata !== undefined && { metadata: this.metadata }),
+            startedAt: startedAt.toISOString(),
+            finishedAt: finishedAt.toISOString(),
+            toRun,
+            saveRun: (target) => writeRun(toRun(), target),
             report: (type, options) => {
                 report({
+                    ...(this.name !== undefined && { name: this.name }),
+                    runId,
                     results: orderedResults.map(reportedCase),
                     aggregated,
                     assertions: assertionResults,
@@ -406,6 +455,7 @@ export class EvalSuite {
         const retrieved = [];
         let reportsToolCalls = false;
         let usage;
+        let lastRun = {};
         const flags = {};
         for (const input of inputs) {
             if (turns.length > 0 && signal.aborted) {
@@ -447,10 +497,12 @@ export class EvalSuite {
                 flags.iterationLimitReached = true;
             if (runResult.status === 'paused')
                 flags.paused = true;
+            lastRun = runIdentityOf(runResult);
             turns.push({
                 input,
                 output,
                 ...(turnCalls && turnCalls.length > 0 && { toolCalls: turnCalls }),
+                ...lastRun,
             });
         }
         return {
@@ -462,6 +514,7 @@ export class EvalSuite {
             ...(toolResults.length > 0 && { toolResults }),
             ...(retrieved.length > 0 && { retrieved }),
             ...(threadId !== undefined && { turns }),
+            ...lastRun,
             ...flags,
         };
     }
@@ -532,6 +585,30 @@ export class EvalSuite {
         return aggregated;
     }
 }
+function runIdentityOf(runResult) {
+    const trace = runResult.trace;
+    const traceId = typeof trace === 'object' && trace !== null
+        ? trace.traceId
+        : undefined;
+    const prompt = RunPromptSchema.safeParse(runResult.prompt);
+    return {
+        ...(typeof traceId === 'string' && traceId !== '' && { traceId }),
+        ...(prompt.success && { prompt: prompt.data }),
+    };
+}
+/** What a run file records of the target: its kind, and an agent's name and model */
+function targetDescription(target) {
+    if (target.fn !== undefined)
+        return { kind: 'fn' };
+    const agent = typeof target.agent === 'object' && target.agent !== null
+        ? target.agent
+        : {};
+    return {
+        kind: 'agent',
+        ...(typeof agent.name === 'string' && { name: agent.name }),
+        ...(typeof agent.model === 'string' && { model: agent.model }),
+    };
+}
 function trialsOf(result) {
     return result.trials ?? [result];
 }
@@ -550,11 +627,11 @@ function failureDetails(settled) {
     const { turns, usage } = settled.error;
     return { ok: false, error: settled.error, turns, ...(usage && { usage }) };
 }
-/** A case result as the reporters get it, the agent's data and the trials kept */
 /** The trial a case is reported by: the first that answered, the first when every trial errored */
 function leadTrial(trials) {
     return trials.find((trial) => trial.error === undefined) ?? trials[0];
 }
+/** A case result as the reporters get it, the agent's data and the trials kept */
 function reportedCase(r) {
     return {
         case: { input: r.case.input, expected: r.case.expected },
@@ -568,6 +645,8 @@ function reportedCase(r) {
         ...(r.toolResults && { toolResults: r.toolResults }),
         ...(r.retrieved && { retrieved: r.retrieved }),
         ...(r.turns && { turns: r.turns }),
+        ...(r.traceId !== undefined && { traceId: r.traceId }),
+        ...(r.prompt && { prompt: r.prompt }),
         scores: r.scores,
         ...(r.trials && { trials: r.trials.map(reportedCase) }),
     };

package/dist/eval-suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAapE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0IrC;;;GAGG;AACH,MAAM,WAAY,SAAQ,KAAK;IAGlB;IACA;IAHX,YACE,KAAc,EACL,KAAuB,EACvB,KAA8B;QAEvC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAH7B,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAyB;QAGvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,6EAA6E;IACrE,eAAe,CAAC,OAAmB;QACzC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,sBAAsB,MAAM,qDAAqD,CAC1G,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,mBAAmB,CAAC,oCAAoC,IAAI,CAAC,MAAM,mBAAmB,CAAC,UAAU,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAmB,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAuB,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACzE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAChF,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnF,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzC,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,SAAuC,CAAC;QAC5C,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAEvF,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;YAC9E,MAAM,KAAK,GAAG,WAAW,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5B,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,SAAS,CACV,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;SAC5B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAwB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAqB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAc,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,KAA8B,CAAC;QACnC,MAAM,KAAK,GAA2E,EAAE,CAAC;QAEzF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,WAAW,GAAqB,EAAE,CAAC;YACzC,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;oBACjD,KAAK;oBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM;oBACN,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;iBACzE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,SAAS;oBAAE,MAAM,KAAK,CAAC;gBACxC,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,KAAK,CACN,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBAClD,CAAC,CAAE,SAAS,CAAC,SAA4B;gBACzC,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,SAAS;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YACjC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;gBAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzD,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;gBAAE,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACjF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;gBAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,MAAM;gBACN,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;aACnE,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,gBAAgB,IAAI,EAAE,SAAS,EAAE,CAAC;YACtC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9C,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,GAAG,KAAK;SACT,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,eAAe,CAAC,OAA2B;QACjD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,GAA0B,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACvE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU;gBAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACxF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAwB;IACxC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,UAKrB;IACC,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACrE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,OAAgB;IACtC,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,kFAAkF;AAClF,mGAAmG;AACnG,SAAS,SAAS,CAAC,MAA0B;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,YAAY,CAAC,CAAmB;IACvC,OAAO;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC"}
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAcpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEzE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAwK1D;;;GAGG;AACH,MAAM,WAAY,SAAQ,KAAK;IAGlB;IACA;IAHX,YACE,KAAc,EACL,KAAuB,EACvB,KAA8B;QAEvC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAH7B,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAyB;QAGvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,UAAU,CAAoC;IAC9C,IAAI,CAAU;IACd,QAAQ,CAAgB;IACxB,SAAS,CAAgB;IAE1C,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,MAAM,UAAU,GAAG,YAAY,CAC7B,uBAAuB,EACvB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAC5C,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QAEpC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;QAEF,IAAI,CAAC,SAAS,GAAG;YACf,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YAC5C,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YACpE,GAAG,CAAC,WAAW,IAAI;gBACjB,KAAK,EAAE;oBACL,KAAK,EAAE,WAAW,CAAC,KAAK;oBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;oBACpC,GAAG,CAAC,WAAW,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC;iBAC3E;aACF,CAAC;YACF,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC;SACvC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,6EAA6E;IACrE,eAAe,CAAC,OAAmB;QACzC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,sBAAsB,MAAM,qDAAqD,CAC1G,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,mBAAmB,CAAC,oCAAoC,IAAI,CAAC,MAAM,mBAAmB,CAAC,UAAU,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAmB,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAuB,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACzE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAChF,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnF,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,GAAwB,CAAC;QAC7B,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,GAAG,KAAK,QAAQ,CAAC;YAChB,KAAK;YACL,SAAS;YACT,UAAU;YACV,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/D,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,SAAS;YACtB,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;SACN,CAAC,CAAC,CAAC;QAEN,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,KAAK;YACL,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/D,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,UAAU,CAAC,WAAW,EAAE;YACpC,KAAK;YACL,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC;YAC9C,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;oBACnD,KAAK;oBACL,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzC,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,SAAuC,CAAC;QAC5C,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAEvF,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;YAC9E,MAAM,KAAK,GAAG,WAAW,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5B,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,SAAS,CACV,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;SAC5B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAwB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAqB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAc,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,KAA8B,CAAC;QACnC,IAAI,OAAO,GAAgB,EAAE,CAAC;QAC9B,MAAM,KAAK,GAA2E,EAAE,CAAC;QAEzF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,WAAW,GAAqB,EAAE,CAAC;YACzC,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;oBACjD,KAAK;oBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM;oBACN,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;iBACzE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,SAAS;oBAAE,MAAM,KAAK,CAAC;gBACxC,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,KAAK,CACN,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBAClD,CAAC,CAAE,SAAS,CAAC,SAA4B;gBACzC,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,SAAS;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YACjC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;gBAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzD,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;gBAAE,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACjF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;gBAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;YACvD,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,MAAM;gBACN,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;gBAClE,GAAG,OAAO;aACX,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,gBAAgB,IAAI,EAAE,SAAS,EAAE,CAAC;YACtC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9C,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,GAAG,OAAO;YACV,GAAG,KAAK;SACT,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,eAAe,CAAC,OAA2B;QACjD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,GAA0B,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACvE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU;gBAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACxF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAQD,SAAS,aAAa,CAAC,SAAyB;IAC9C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC9B,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QACzC,CAAC,CAAE,KAA+B,CAAC,OAAO;QAC1C,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QACjE,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,IAAI,MAAM,CAAC,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACnD,MAAM,KAAK,GACT,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QACvD,CAAC,CAAE,MAAM,CAAC,KAA6C;QACvD,CAAC,CAAC,EAAE,CAAC;IACT,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;QAC3D,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;KAC/D,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,MAAwB;IACxC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,UAKrB;IACC,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACrE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,OAAgB;IACtC,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,mGAAmG;AACnG,SAAS,SAAS,CAAC,MAA0B;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,kFAAkF;AAClF,SAAS,YAAY,CAAC,CAAmB;IACvC,OAAO;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QACtD,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -22,7 +22,7 @@ export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCalls
 export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
 export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
-export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
+export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, EvalPrompt, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
 export type { CustomMetricConfig, CustomMetricData } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
@@ -31,6 +31,10 @@ export { maxErrorRate } from './assertions/index.js';
 export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export type { ReporterType, ReporterOptions, ReportedCase } from './reporters/index.js';
+export { reportComparison, comparisonMarkdown, comparisonHtml } from './reporters/comparison.js';
+export type { ReportableComparison, ComparisonReporterType } from './reporters/comparison.js';
+export { loadRun, caseKey, RunStore, compareRuns, EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION, } from './runs/index.js';
+export type { EvalRun, EvalRunCase, EvalRunConfig, EvalRunSource, JsonMetadata, RunSummary, RunFilter, RunStoreOptions, SkippedRunFile, CompareRunsOptions, RunComparison, RunMetricComparison, RunCaseChange, RunCaseRef, RunRef, RunWinner, } from './runs/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export type { TTestResult } from './stats/t-test.js';
 export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,eAAe,EACf,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,YAAY,EACZ,UAAU,EACV,UAAU,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,eAAe,EACf,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,YAAY,EACZ,UAAU,EACV,UAAU,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC9F,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAE3F,OAAO,EACL,OAAO,EACP,OAAO,EACP,QAAQ,EACR,WAAW,EACX,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,MAAM,QAAQ,CAAC;AAChB,YAAY,EACV,OAAO,EACP,WAAW,EACX,aAAa,EACb,aAAa,EACb,YAAY,EACZ,UAAU,EACV,SAAS,EACT,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,aAAa,EACb,UAAU,EACV,MAAM,EACN,SAAS,GACV,MAAM,QAAQ,CAAC;AAEhB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}