@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/eval-suite.d.ts

@@ -5,6 +5,7 @@ import type { MetricFn, Passage, ScoredCaseResult, StatisticalMetricFn } from '.
 import type { LLMMetricFn } from './metrics/llm-judge.js';
 import type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 import type { ReporterType, ReporterOptions } from './reporters/index.js';
+import type { EvalRun, JsonMetadata } from './runs/schema.js';
 /** A turn of a conversation: what the user said and what the target answered */
 export interface EvalTurn {
     input: string;
@@ -52,6 +53,13 @@ export interface EvalProgress {
 export interface EvalSuiteOptions {
     dataset: Dataset;
     target: EvalTarget;
+    /** Names the experiment the suite runs, such as `support-agent`, in reports and run files */
+    name?: string;
+    /**
+     * Describes the experiment in run files, such as `{ branch: 'main', commit: 'a1b2c3' }`. Must be
+     * JSON-serializable: keys holding `undefined` are dropped, other values JSON cannot hold throw
+     */
+    metadata?: Record<string, unknown>;
     metrics?: MetricFn[];
     statisticalMetrics?: StatisticalMetricFn[];
     judge?: JudgeConfigInput;
@@ -97,6 +105,28 @@ export interface EvalSuiteResult {
     report: (type: ReporterType | ReporterType[], options?: ReporterOptions) => void;
     /** Writes the mean of each metric as JSON; throws when a mean is not a finite number */
     saveBaseline: (path: string) => void;
+    /** Unique id of the run, which names its run file: `20261022T101530123Z-1a2b3c4d` */
+    runId: string;
+    /** The suite's `name` */
+    name?: string;
+    /** The suite's `metadata` */
+    metadata?: JsonMetadata;
+    /** When the run started, ISO 8601 */
+    startedAt: string;
+    /** When the run finished, ISO 8601 */
+    finishedAt: string;
+    /**
+     * The run as a run file holds it: id, times, name and metadata, the dataset fingerprint, the
+     * suite's configuration (never its functions, Cogitator or keys), the prompts used, every case
+     * result with its `caseKey`, the aggregates, assertions and stats
+     */
+    toRun: () => EvalRun;
+    /**
+     * Writes the run file and returns its path: `<target>/<runId>.json` when `target` is a
+     * directory or ends with a path separator, `target` itself otherwise. Read it back with
+     * `loadRun()`, keep runs with a `RunStore` and compare them with `compareRuns()`
+     */
+    saveRun: (target: string) => string;
 }
 export declare function isLLMMetric(m: MetricFn): m is LLMMetricFn;
 export declare class EvalSuite {
@@ -110,6 +140,9 @@ export declare class EvalSuite {
     private readonly retries;
     private readonly trials;
     private readonly onProgress?;
+    private readonly name?;
+    private readonly metadata?;
+    private readonly runConfig;
     constructor(opts: EvalSuiteOptions);
     private validateTarget;
     /** A `passAtK()` metric needs the metric it reads and at least `k` trials */

package/dist/eval-suite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAMR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE/E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAyDD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA6ClC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAyGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YA4ElB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAOR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG/E,OAAO,KAAK,EAAE,OAAO,EAAiB,YAAY,EAAE,MAAM,eAAe,CAAC;AAG1E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC;IACrB;;;;OAIG;IACH,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACrC;AA2DD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;IAC/D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;gBAE9B,IAAI,EAAE,gBAAgB;IAoElC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAqIjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YAgFlB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}

package/dist/eval-suite.js

@@ -7,6 +7,9 @@ import { ragSearchPassages } from './metrics/retrieval.js';
 import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
 import { aggregate, mean } from './stats/index.js';
 import { report } from './reporters/index.js';
+import { parseOptions } from './options.js';
+import { ExperimentOptionsSchema, RunPromptSchema } from './runs/schema.js';
+import { buildRun, newRunId, writeRun } from './runs/run.js';
 /**
  * A turn of a multi-turn case failed: the error of the turn, with the turns run so far (the
  * failed one last) and what they spent
@@ -118,6 +121,9 @@ export class EvalSuite {
     retries;
     trials;
     onProgress;
+    name;
+    metadata;
+    runConfig;
     constructor(opts) {
         const config = EvalSuiteConfigSchema.parse({
             concurrency: opts.concurrency,
@@ -134,6 +140,9 @@ export class EvalSuite {
         this.retries = config.retries;
         this.trials = config.trials;
         this.onProgress = opts.onProgress;
+        const experiment = parseOptions(ExperimentOptionsSchema, { name: opts.name, metadata: opts.metadata }, 'EvalSuite');
+        this.name = experiment.name;
+        this.metadata = experiment.metadata;
         this.validateTarget();
         const rawMetrics = opts.metrics ?? [];
         this.validateMetricNames(rawMetrics);
@@ -149,6 +158,20 @@ export class EvalSuite {
         }
         const judge = judgeConfig && judgeCogitator ? judgeContextFor(judgeCogitator, judgeConfig) : null;
         this.boundMetrics = rawMetrics.map((m) => isLLMMetric(m) && judge ? bindJudgeContext(m, judge) : m);
+        this.runConfig = {
+            metrics: rawMetrics.map((m) => m.metricName),
+            statisticalMetrics: this.statisticalMetrics.map((m) => m.metricName),
+            ...(judgeConfig && {
+                judge: {
+                    model: judgeConfig.model,
+                    temperature: judgeConfig.temperature,
+                    ...(judgeConfig.samples !== undefined && { samples: judgeConfig.samples }),
+                },
+            }),
+            trials: this.trials,
+            concurrency: this.concurrency,
+            target: targetDescription(this.target),
+        };
     }
     validateTarget() {
         const { fn, agent, cogitator } = this.target;
@@ -209,7 +232,9 @@ export class EvalSuite {
             throw new Error(`maxCases must be a positive integer, got ${maxCases}`);
         }
         signal?.throwIfAborted();
-        const suiteStart = Date.now();
+        const startedAt = new Date();
+        const runId = newRunId(startedAt);
+        const suiteStart = startedAt.getTime();
         const cases = this.cases(maxCases);
         const total = cases.length;
         const trials = this.trials;
@@ -255,13 +280,37 @@ export class EvalSuite {
             judgeCost,
         };
         const assertionResults = this.assertionFns.map((fn) => fn(aggregated, stats));
+        const finishedAt = new Date(suiteStart + stats.duration);
+        let run;
+        const toRun = () => (run ??= buildRun({
+            runId,
+            startedAt,
+            finishedAt,
+            ...(this.name !== undefined && { name: this.name }),
+            ...(this.metadata !== undefined && { metadata: this.metadata }),
+            cases,
+            config: this.runConfig,
+            results: orderedResults,
+            aggregated,
+            assertions: assertionResults,
+            stats,
+        }));
         const suiteResult = {
             results: orderedResults,
             aggregated,
             assertions: assertionResults,
             stats,
+            runId,
+            ...(this.name !== undefined && { name: this.name }),
+            ...(this.metadata !== undefined && { metadata: this.metadata }),
+            startedAt: startedAt.toISOString(),
+            finishedAt: finishedAt.toISOString(),
+            toRun,
+            saveRun: (target) => writeRun(toRun(), target),
             report: (type, options) => {
                 report({
+                    ...(this.name !== undefined && { name: this.name }),
+                    runId,
                     results: orderedResults.map(reportedCase),
                     aggregated,
                     assertions: assertionResults,
@@ -406,6 +455,7 @@ export class EvalSuite {
         const retrieved = [];
         let reportsToolCalls = false;
         let usage;
+        let lastRun = {};
         const flags = {};
         for (const input of inputs) {
             if (turns.length > 0 && signal.aborted) {
@@ -447,10 +497,12 @@ export class EvalSuite {
                 flags.iterationLimitReached = true;
             if (runResult.status === 'paused')
                 flags.paused = true;
+            lastRun = runIdentityOf(runResult);
             turns.push({
                 input,
                 output,
                 ...(turnCalls && turnCalls.length > 0 && { toolCalls: turnCalls }),
+                ...lastRun,
             });
         }
         return {
@@ -462,6 +514,7 @@ export class EvalSuite {
             ...(toolResults.length > 0 && { toolResults }),
             ...(retrieved.length > 0 && { retrieved }),
             ...(threadId !== undefined && { turns }),
+            ...lastRun,
             ...flags,
         };
     }
@@ -532,6 +585,30 @@ export class EvalSuite {
         return aggregated;
     }
 }
+function runIdentityOf(runResult) {
+    const trace = runResult.trace;
+    const traceId = typeof trace === 'object' && trace !== null
+        ? trace.traceId
+        : undefined;
+    const prompt = RunPromptSchema.safeParse(runResult.prompt);
+    return {
+        ...(typeof traceId === 'string' && traceId !== '' && { traceId }),
+        ...(prompt.success && { prompt: prompt.data }),
+    };
+}
+/** What a run file records of the target: its kind, and an agent's name and model */
+function targetDescription(target) {
+    if (target.fn !== undefined)
+        return { kind: 'fn' };
+    const agent = typeof target.agent === 'object' && target.agent !== null
+        ? target.agent
+        : {};
+    return {
+        kind: 'agent',
+        ...(typeof agent.name === 'string' && { name: agent.name }),
+        ...(typeof agent.model === 'string' && { model: agent.model }),
+    };
+}
 function trialsOf(result) {
     return result.trials ?? [result];
 }
@@ -550,11 +627,11 @@ function failureDetails(settled) {
     const { turns, usage } = settled.error;
     return { ok: false, error: settled.error, turns, ...(usage && { usage }) };
 }
-/** A case result as the reporters get it, the agent's data and the trials kept */
 /** The trial a case is reported by: the first that answered, the first when every trial errored */
 function leadTrial(trials) {
     return trials.find((trial) => trial.error === undefined) ?? trials[0];
 }
+/** A case result as the reporters get it, the agent's data and the trials kept */
 function reportedCase(r) {
     return {
         case: { input: r.case.input, expected: r.case.expected },
@@ -568,6 +645,8 @@ function reportedCase(r) {
         ...(r.toolResults && { toolResults: r.toolResults }),
         ...(r.retrieved && { retrieved: r.retrieved }),
         ...(r.turns && { turns: r.turns }),
+        ...(r.traceId !== undefined && { traceId: r.traceId }),
+        ...(r.prompt && { prompt: r.prompt }),
         scores: r.scores,
         ...(r.trials && { trials: r.trials.map(reportedCase) }),
     };

package/dist/eval-suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAapE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0IrC;;;GAGG;AACH,MAAM,WAAY,SAAQ,KAAK;IAGlB;IACA;IAHX,YACE,KAAc,EACL,KAAuB,EACvB,KAA8B;QAEvC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAH7B,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAyB;QAGvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,6EAA6E;IACrE,eAAe,CAAC,OAAmB;QACzC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,sBAAsB,MAAM,qDAAqD,CAC1G,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,mBAAmB,CAAC,oCAAoC,IAAI,CAAC,MAAM,mBAAmB,CAAC,UAAU,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAmB,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAuB,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACzE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAChF,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnF,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzC,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,SAAuC,CAAC;QAC5C,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAEvF,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;YAC9E,MAAM,KAAK,GAAG,WAAW,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5B,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,SAAS,CACV,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;SAC5B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAwB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAqB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAc,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,KAA8B,CAAC;QACnC,MAAM,KAAK,GAA2E,EAAE,CAAC;QAEzF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,WAAW,GAAqB,EAAE,CAAC;YACzC,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;oBACjD,KAAK;oBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM;oBACN,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;iBACzE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,SAAS;oBAAE,MAAM,KAAK,CAAC;gBACxC,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,KAAK,CACN,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBAClD,CAAC,CAAE,SAAS,CAAC,SAA4B;gBACzC,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,SAAS;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YACjC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;gBAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzD,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;gBAAE,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACjF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;gBAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,MAAM;gBACN,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;aACnE,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,gBAAgB,IAAI,EAAE,SAAS,EAAE,CAAC;YACtC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9C,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,GAAG,KAAK;SACT,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,eAAe,CAAC,OAA2B;QACjD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,GAA0B,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACvE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU;gBAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACxF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAwB;IACxC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,UAKrB;IACC,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACrE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,OAAgB;IACtC,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,kFAAkF;AAClF,mGAAmG;AACnG,SAAS,SAAS,CAAC,MAA0B;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,YAAY,CAAC,CAAmB;IACvC,OAAO;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC"}
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAcpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEzE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAwK1D;;;GAGG;AACH,MAAM,WAAY,SAAQ,KAAK;IAGlB;IACA;IAHX,YACE,KAAc,EACL,KAAuB,EACvB,KAA8B;QAEvC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAH7B,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAyB;QAGvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,UAAU,CAAoC;IAC9C,IAAI,CAAU;IACd,QAAQ,CAAgB;IACxB,SAAS,CAAgB;IAE1C,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,MAAM,UAAU,GAAG,YAAY,CAC7B,uBAAuB,EACvB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAC5C,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QAEpC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;QAEF,IAAI,CAAC,SAAS,GAAG;YACf,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YAC5C,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YACpE,GAAG,CAAC,WAAW,IAAI;gBACjB,KAAK,EAAE;oBACL,KAAK,EAAE,WAAW,CAAC,KAAK;oBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;oBACpC,GAAG,CAAC,WAAW,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC;iBAC3E;aACF,CAAC;YACF,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC;SACvC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,6EAA6E;IACrE,eAAe,CAAC,OAAmB;QACzC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,sBAAsB,MAAM,qDAAqD,CAC1G,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,CAAC,UAAU,mBAAmB,CAAC,oCAAoC,IAAI,CAAC,MAAM,mBAAmB,CAAC,UAAU,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAmB,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAuB,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACzE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAChF,CAAC;QACF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnF,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,GAAwB,CAAC;QAC7B,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,GAAG,KAAK,QAAQ,CAAC;YAChB,KAAK;YACL,SAAS;YACT,UAAU;YACV,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/D,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,SAAS;YACtB,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;SACN,CAAC,CAAC,CAAC;QAEN,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,KAAK;YACL,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/D,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,UAAU,CAAC,WAAW,EAAE;YACpC,KAAK;YACL,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC;YAC9C,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;oBACnD,KAAK;oBACL,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzC,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,SAAuC,CAAC;QAC5C,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;SACvC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAEvF,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;YAC9E,MAAM,KAAK,GAAG,WAAW,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5B,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,SAAS,CACV,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;SAC5B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAwB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAqB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAc,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,KAA8B,CAAC;QACnC,IAAI,OAAO,GAAgB,EAAE,CAAC;QAC9B,MAAM,KAAK,GAA2E,EAAE,CAAC;QAEzF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,WAAW,GAAqB,EAAE,CAAC;YACzC,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;oBACjD,KAAK;oBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM;oBACN,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;iBACzE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,SAAS;oBAAE,MAAM,KAAK,CAAC;gBACxC,MAAM,IAAI,WAAW,CACnB,KAAK,EACL,CAAC,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAC7D,KAAK,CACN,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBAClD,CAAC,CAAE,SAAS,CAAC,SAA4B;gBACzC,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,SAAS;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YACjC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;gBAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzD,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;gBAAE,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACjF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;gBAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;YACvD,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,MAAM;gBACN,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;gBAClE,GAAG,OAAO;aACX,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACtC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,gBAAgB,IAAI,EAAE,SAAS,EAAE,CAAC;YACtC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9C,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,GAAG,OAAO;YACV,GAAG,KAAK;SACT,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,eAAe,CAAC,OAA2B;QACjD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,GAA0B,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACvE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU;gBAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACxF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAQD,SAAS,aAAa,CAAC,SAAyB;IAC9C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC9B,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QACzC,CAAC,CAAE,KAA+B,CAAC,OAAO;QAC1C,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QACjE,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,IAAI,MAAM,CAAC,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACnD,MAAM,KAAK,GACT,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QACvD,CAAC,CAAE,MAAM,CAAC,KAA6C;QACvD,CAAC,CAAC,EAAE,CAAC;IACT,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;QAC3D,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;KAC/D,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,MAAwB;IACxC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,UAKrB;IACC,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACrE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,cAAc,CAAC,OAAgB;IACtC,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,mGAAmG;AACnG,SAAS,SAAS,CAAC,MAA0B;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,kFAAkF;AAClF,SAAS,YAAY,CAAC,CAAmB;IACvC,OAAO;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QACtD,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -2,14 +2,19 @@ export declare const VERSION: string;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
 export type { EvalTarget, EvalTargetContext, EvalTurn, RetrievalToolResult, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
-export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ComparisonResult, } from './eval-comparison.js';
+export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ResourceComparison, ComparisonResult, ComparisonTest, ComparisonWinner, } from './eval-comparison.js';
+export type { PairwiseOptions, PairwiseResult, PairwiseCase, PairwiseVerdict, PairwiseSide, PairwiseOutcome, } from './pairwise.js';
+export { calibrateJudge } from './calibrate-judge.js';
+export type { CalibrationExample, CalibrateJudgeOptions, CalibrationCase, JudgeCalibration, } from './calibrate-judge.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
 export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
-export type { LLMMetricFn, JudgeContext, JudgeMessage } from './metrics/llm-judge.js';
+export type { LLMMetricFn, LLMMetricOptions, PromptMetricOptions, RubricMetricOptions, RubricLevel, JudgeContext, JudgeMessage, JudgeRunOptions, JudgeFormat, JudgeVerdict, JudgeScore, JudgeSetup, } from './metrics/llm-judge.js';
+export { gEval } from './metrics/g-eval.js';
+export type { GEvalOptions } from './metrics/g-eval.js';
 export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
 export type { PassAtKOptions, PassAtKMetricFn } from './metrics/statistical.js';
 export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
@@ -17,7 +22,7 @@ export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCalls
 export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
 export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
-export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
+export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, EvalPrompt, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
 export type { CustomMetricConfig, CustomMetricData } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
@@ -26,10 +31,21 @@ export { maxErrorRate } from './assertions/index.js';
 export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export type { ReporterType, ReporterOptions, ReportedCase } from './reporters/index.js';
+export { reportComparison, comparisonMarkdown, comparisonHtml } from './reporters/comparison.js';
+export type { ReportableComparison, ComparisonReporterType } from './reporters/comparison.js';
+export { loadRun, caseKey, RunStore, compareRuns, EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION, } from './runs/index.js';
+export type { EvalRun, EvalRunCase, EvalRunConfig, EvalRunSource, JsonMetadata, RunSummary, RunFilter, RunStoreOptions, SkippedRunFile, CompareRunsOptions, RunComparison, RunMetricComparison, RunCaseChange, RunCaseRef, RunRef, RunWinner, } from './runs/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export type { TTestResult } from './stats/t-test.js';
-export { mcnemarsTest } from './stats/mcnemar.js';
-export type { McNemarResult } from './stats/mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';
+export type { McNemarResult, McNemarExactResult } from './stats/mcnemar.js';
+export { wilcoxonSignedRank } from './stats/wilcoxon.js';
+export type { WilcoxonResult } from './stats/wilcoxon.js';
+export { bootstrapCI } from './stats/bootstrap.js';
+export type { BootstrapOptions, BootstrapInterval } from './stats/bootstrap.js';
+export { cohensD } from './stats/effect-size.js';
+export { holm } from './stats/holm.js';
+export { pearson, spearman } from './stats/correlation.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
 export type { EvalCase, ExpectedToolCall, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
 export { createRunEvalTool, evalTools } from './tools.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,eAAe,EACf,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,YAAY,EACZ,UAAU,EACV,UAAU,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC9F,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAE3F,OAAO,EACL,OAAO,EACP,OAAO,EACP,QAAQ,EACR,WAAW,EACX,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,MAAM,QAAQ,CAAC;AAChB,YAAY,EACV,OAAO,EACP,WAAW,EACX,aAAa,EACb,aAAa,EACb,YAAY,EACZ,UAAU,EACV,SAAS,EACT,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,aAAa,EACb,UAAU,EACV,MAAM,EACN,SAAS,GACV,MAAM,QAAQ,CAAC;AAEhB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -3,12 +3,14 @@ const packageJson = createRequire(import.meta.url)('../package.json');
 export const VERSION = packageJson.version;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
+export { calibrateJudge } from './calibrate-judge.js';
 export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
 export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export { gEval } from './metrics/g-eval.js';
 export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
 export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
 export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
@@ -19,8 +21,15 @@ export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
 export { maxErrorRate } from './assertions/index.js';
 export { report } from './reporters/index.js';
+export { reportComparison, comparisonMarkdown, comparisonHtml } from './reporters/comparison.js';
+export { loadRun, caseKey, RunStore, compareRuns, EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION, } from './runs/index.js';
 export { pairedTTest } from './stats/t-test.js';
-export { mcnemarsTest } from './stats/mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';
+export { wilcoxonSignedRank } from './stats/wilcoxon.js';
+export { bootstrapCI } from './stats/bootstrap.js';
+export { cohensD } from './stats/effect-size.js';
+export { holm } from './stats/holm.js';
+export { pearson, spearman } from './stats/correlation.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
 export { createRunEvalTool, evalTools } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAkBnD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAe7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAkB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAG9F,OAAO,EACL,OAAO,EACP,OAAO,EACP,QAAQ,EACR,WAAW,EACX,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,MAAM,QAAQ,CAAC;AAoBhB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/g-eval.d.ts

@@ -0,0 +1,29 @@
+import type { LLMMetricFn } from './llm-judge.js';
+export interface GEvalOptions {
+    name: string;
+    /** What a good response does, in your words */
+    criteria: string;
+    /** The evaluation steps the judge follows. Default: the judge writes them from `criteria` */
+    steps?: readonly string[];
+    /** The scale the judge scores on, `[min, max]`, normalized to 0 - 1. Default `[1, 5]` */
+    scale?: readonly [number, number];
+}
+/**
+ * The steps of a numbered or bulleted list, in order: `1. ...`, `1) ...`, `Step 1: ...`,
+ * `- ...`, a Markdown bold marker (`**1.** ...`, `**Step 1:** ...`) included. A bullet needs a
+ * space after it, so a rule (`---`) or emphasis (`*note*`) is no step, and the text of a step is
+ * kept as written
+ */
+export declare function stepsOf(text: string): string[];
+/**
+ * G-Eval: the judge scores each case on `scale` following evaluation steps, the score normalized
+ * to 0 - 1. Without `steps`, the judge writes them from `criteria` once per metric instance,
+ * before the first case it scores, and every later case reuses them. A failed generation scores
+ * the cases waiting on it as judge errors, and the next case tries again.
+ *
+ * The original G-Eval weights each score of the scale by the probability of its token, which not
+ * every provider exposes. Set `judge.samples` (with a `temperature` above 0) to average several
+ * verdicts instead.
+ */
+export declare function gEval(opts: GEvalOptions): LLMMetricFn;
+//# sourceMappingURL=g-eval.d.ts.map

package/dist/metrics/g-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"g-eval.d.ts","sourceRoot":"","sources":["../../src/metrics/g-eval.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAsD,WAAW,EAAE,MAAM,aAAa,CAAC;AAEnG,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,6FAA6F;IAC7F,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,yFAAyF;IACzF,KAAK,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAkBD;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAK9C;AAaD;;;;;;;;;GASG;AACH,wBAAgB,KAAK,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,CAiDrD"}

package/dist/metrics/g-eval.js

@@ -0,0 +1,89 @@
+import { z } from 'zod';
+import { parseOptions } from '../options.js';
+import { JudgeUsageError, createJudgeMetric, usageOf } from './llm-judge.js';
+const GEvalOptionsSchema = z.object({
+    name: z.string().trim().min(1, 'name must not be empty'),
+    criteria: z.string().trim().min(1, 'criteria must not be empty'),
+    steps: z
+        .array(z.string().trim().min(1, 'a step must not be empty'))
+        .min(1, 'give at least one step, or leave steps out for the judge to write them')
+        .optional(),
+    scale: z
+        .tuple([z.number(), z.number()])
+        .refine(([min, max]) => min < max, 'scale must be [min, max] with min below max')
+        .default([1, 5]),
+});
+const STEP_LINE = /^\s*(?:(\*\*|__)?(?:step\s*)?\d+\s*[.):-]\1?\s*|(?:\*\*|__)?[-*\u2022](?:\*\*|__)?\s+)(.*\S)\s*$/i;
+/**
+ * The steps of a numbered or bulleted list, in order: `1. ...`, `1) ...`, `Step 1: ...`,
+ * `- ...`, a Markdown bold marker (`**1.** ...`, `**Step 1:** ...`) included. A bullet needs a
+ * space after it, so a rule (`---`) or emphasis (`*note*`) is no step, and the text of a step is
+ * kept as written
+ */
+export function stepsOf(text) {
+    return text
+        .split('\n')
+        .map((line) => STEP_LINE.exec(line)?.[2])
+        .filter((step) => step !== undefined);
+}
+const hasSteps = (output) => stepsOf(output).length > 0;
+function stepsRequest(criteria) {
+    return `Criteria for evaluating an assistant's response to an input:\n${criteria}\n\nWrite the evaluation steps a careful grader follows to score a response on these criteria, 3 to 6 short steps. Reply with a numbered list, one step per line, and nothing else.`;
+}
+function scoringPrompt(criteria, steps, [min, max]) {
+    const numbered = steps.map((step, i) => `${i + 1}. ${step}`).join('\n');
+    return `You are evaluating an assistant's response on the criteria below.\n\nCriteria:\n${criteria}\n\nEvaluation steps:\n${numbered}\n\nFollow the evaluation steps, then score the <response> from ${min} (does not meet the criteria at all) to ${max} (fully meets them).\n\nRespond with JSON: {"score": <number from ${min} to ${max}>, "reasoning": "<explanation>"}`;
+}
+/**
+ * G-Eval: the judge scores each case on `scale` following evaluation steps, the score normalized
+ * to 0 - 1. Without `steps`, the judge writes them from `criteria` once per metric instance,
+ * before the first case it scores, and every later case reuses them. A failed generation scores
+ * the cases waiting on it as judge errors, and the next case tries again.
+ *
+ * The original G-Eval weights each score of the scale by the probability of its token, which not
+ * every provider exposes. Set `judge.samples` (with a `temperature` above 0) to average several
+ * verdicts instead.
+ */
+export function gEval(opts) {
+    const { name, criteria, steps, scale } = parseOptions(GEvalOptionsSchema, opts, 'gEval');
+    const [min, max] = scale;
+    let generating;
+    const generate = async (context) => {
+        const answered = await context.cogitator.run({
+            input: stepsRequest(criteria),
+            format: 'text',
+            accept: hasSteps,
+        });
+        const usage = usageOf(answered.usage);
+        const written = stepsOf(answered.output);
+        if (written.length === 0) {
+            throw new JudgeUsageError('could not read evaluation steps from the judge, it gave no numbered list', usage);
+        }
+        return { systemPrompt: scoringPrompt(criteria, written, scale), ...(usage && { usage }) };
+    };
+    const setup = async (context) => {
+        if (steps)
+            return { systemPrompt: scoringPrompt(criteria, steps, scale) };
+        if (generating) {
+            try {
+                return { systemPrompt: (await generating).systemPrompt };
+            }
+            catch (error) {
+                throw new Error(error instanceof Error ? error.message : String(error), { cause: error });
+            }
+        }
+        const started = generate(context).catch((error) => {
+            if (generating === started)
+                generating = undefined;
+            const message = error instanceof Error ? error.message : String(error);
+            throw new JudgeUsageError(`gEval could not generate its evaluation steps: ${message}`, error instanceof JudgeUsageError ? error.usage : undefined, { cause: error });
+        });
+        generating = started;
+        return started;
+    };
+    const score = ({ score: raw }) => ({
+        score: (Math.max(min, Math.min(max, raw)) - min) / (max - min),
+    });
+    return createJudgeMetric(name, scoringPrompt(criteria, steps ?? [], scale), { setup, score });
+}
+//# sourceMappingURL=g-eval.js.map

package/dist/metrics/g-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"g-eval.js","sourceRoot":"","sources":["../../src/metrics/g-eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAa1E,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC;IACxD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4BAA4B,CAAC;IAChE,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;SAC3D,GAAG,CAAC,CAAC,EAAE,wEAAwE,CAAC;SAChF,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;SAC/B,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,6CAA6C,CAAC;SAChF,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;CACnB,CAAC,CAAC;AAEH,MAAM,SAAS,GACb,mGAAmG,CAAC;AAEtG;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAEhE,SAAS,YAAY,CAAC,QAAgB;IACpC,OAAO,iEAAiE,QAAQ,qLAAqL,CAAC;AACxQ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB,EAAE,KAAwB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAmB;IAC7F,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxE,OAAO,mFAAmF,QAAQ,0BAA0B,QAAQ,mEAAmE,GAAG,2CAA2C,GAAG,qEAAqE,GAAG,OAAO,GAAG,kCAAkC,CAAC;AAC/W,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,KAAK,CAAC,IAAkB;IACtC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,CAAC,kBAAkB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzF,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC;IACzB,IAAI,UAA2C,CAAC;IAEhD,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAqB,EAAuB,EAAE;QACpE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC;YAC3C,KAAK,EAAE,YAAY,CAAC,QAAQ,CAAC;YAC7B,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,QAAQ;SACjB,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,eAAe,CACvB,0EAA0E,EAC1E,KAAK,CACN,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC5F,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,EAAE,OAAqB,EAAuB,EAAE;QACjE,IAAI,KAAK;YAAE,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;QAC1E,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,OAAO,EAAE,YAAY,EAAE,CAAC,MAAM,UAAU,CAAC,CAAC,YAAY,EAAE,CAAC;YAC3D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACzD,IAAI,UAAU,KAAK,OAAO;gBAAE,UAAU,GAAG,SAAS,CAAC;YACnD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,IAAI,eAAe,CACvB,kDAAkD,OAAO,EAAE,EAC3D,KAAK,YAAY,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAC1D,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,UAAU,GAAG,OAAO,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAgB,EAAc,EAAE,CAAC,CAAC;QAC3D,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;KAC/D,CAAC,CAAC;IAEH,OAAO,iBAAiB,CAAC,IAAI,EAAE,aAAa,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAChG,CAAC"}