@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/metrics/types.d.ts

@@ -39,11 +39,26 @@ export interface Passage {
     score?: number;
     source?: string;
 }
+/** The versioned instructions an agent run used, as its `RunResult.prompt` reports them */
+export interface EvalPrompt {
+    /** The agent's `id` when it was set explicitly, else its `name` */
+    key: string;
+    versionId?: string;
+    version?: number;
+    abTest?: {
+        id: string;
+        variant: 'control' | 'treatment';
+    };
+}
 /** One user turn of a multi-turn case and the answer to it */
 export interface EvalTurnResult {
     input: string;
     output: string;
     toolCalls?: readonly EvalToolCall[];
+    /** The trace of the agent run that answered the turn */
+    traceId?: string;
+    /** The versioned instructions the agent run of the turn used */
+    prompt?: EvalPrompt;
     /** Set on the turn that failed, the last one the case ran */
     error?: string;
 }
@@ -74,6 +89,16 @@ export interface EvalCaseResult {
     retrieved?: readonly Passage[];
     /** Each turn of a case with `turns`: `input` first, then the further turns */
     turns?: readonly EvalTurnResult[];
+    /**
+     * The trace of the agent run, from its `RunResult.trace.traceId`, to find the run in a trace
+     * viewer. With `turns`, the last turn's. A `fn` target has none
+     */
+    traceId?: string;
+    /**
+     * The versioned instructions the agent run used, from its `RunResult.prompt`. With `turns`, the
+     * last turn's. Set when the Cogitator versions instructions (`cogitator.prompts`)
+     */
+    prompt?: EvalPrompt;
     /**
      * Every run of the case when the suite runs several `trials`, each with its own scores. The
      * case result itself is the first of them

package/dist/metrics/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,2FAA2F;AAC3F,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,SAAS,GAAG,WAAW,CAAA;KAAE,CAAC;CAC3D;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}

package/dist/metrics/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA4HA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAmJA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}

package/dist/options.d.ts

@@ -0,0 +1,4 @@
+import type { z } from 'zod';
+/** Parses options with a schema, throwing one readable error that names what is wrong */
+export declare function parseOptions<Schema extends z.ZodType>(schema: Schema, value: unknown, label: string): z.output<Schema>;
+//# sourceMappingURL=options.d.ts.map

package/dist/options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,yFAAyF;AACzF,wBAAgB,YAAY,CAAC,MAAM,SAAS,CAAC,CAAC,OAAO,EACnD,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,GACZ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAOlB"}

package/dist/options.js

@@ -0,0 +1,9 @@
+/** Parses options with a schema, throwing one readable error that names what is wrong */
+export function parseOptions(schema, value, label) {
+    const parsed = schema.safeParse(value);
+    if (parsed.success)
+        return parsed.data;
+    const issues = parsed.error.issues.map((issue) => issue.path.length > 0 ? `${issue.path.join('.')}: ${issue.message}` : issue.message);
+    throw new Error(`${label}: ${issues.join(', ')}`);
+}
+//# sourceMappingURL=options.js.map

package/dist/options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"options.js","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AAEA,yFAAyF;AACzF,MAAM,UAAU,YAAY,CAC1B,MAAc,EACd,KAAc,EACd,KAAa;IAEb,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CACpF,CAAC;IACF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACpD,CAAC"}

package/dist/pairwise.d.ts

@@ -0,0 +1,53 @@
+import type { EvalSuiteResult } from './eval-suite.js';
+import type { JudgeContext } from './metrics/llm-judge.js';
+export interface PairwiseOptions {
+    /** What makes one answer better than the other. Default: correct, relevant, complete, clear */
+    criteria?: string;
+    /** The name the result reports. Default `pairwise` */
+    name?: string;
+}
+export type PairwiseSide = 'baseline' | 'challenger';
+export type PairwiseOutcome = PairwiseSide | 'tie';
+/** One of the two verdicts on a case */
+export interface PairwiseVerdict {
+    /** The side shown to the judge first, as answer A */
+    first: PairwiseSide;
+    /** The side the judge preferred, or `tie` */
+    preferred: PairwiseOutcome;
+    reasoning?: string;
+}
+export interface PairwiseCase {
+    /** Position of the case in the dataset */
+    index: number;
+    input: string;
+    winner: PairwiseOutcome;
+    /** The verdict with the baseline first, then the one with the challenger first */
+    verdicts: PairwiseVerdict[];
+    /** The side that errored and so lost the case without a judge call */
+    errored?: PairwiseSide;
+    /** Why the judge gave no verdict in one of the orders, which makes the case a tie */
+    error?: string;
+}
+export interface PairwiseResult {
+    name: string;
+    wins: {
+        baseline: number;
+        challenger: number;
+    };
+    ties: number;
+    /** Cases compared: those at least one side answered */
+    cases: number;
+    /** Challenger wins over the decided cases (wins of either side), NaN when none was decided */
+    winRate: number;
+    /** Cases the judge failed on in at least one order, counted as ties */
+    judgeErrors: number;
+    /** USD the judge calls cost, as far as the judge reported usage */
+    cost: number;
+    perCase: PairwiseCase[];
+}
+/**
+ * Asks the judge which side answered each case better, see `EvalComparisonOptions.pairwise`.
+ * Cases both sides errored on are left out
+ */
+export declare function judgePairwise(baseline: EvalSuiteResult, challenger: EvalSuiteResult, judge: JudgeContext, options: PairwiseOptions, concurrency: number, signal?: AbortSignal): Promise<PairwiseResult>;
+//# sourceMappingURL=pairwise.d.ts.map

package/dist/pairwise.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.ts","sourceRoot":"","sources":["../src/pairwise.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAUxD,MAAM,WAAW,eAAe;IAC9B,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,KAAK,CAAC;AAEnD,wCAAwC;AACxC,MAAM,WAAW,eAAe;IAC9B,qDAAqD;IACrD,KAAK,EAAE,YAAY,CAAC;IACpB,6CAA6C;IAC7C,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,kFAAkF;IAClF,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,sEAAsE;IACtE,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,8FAA8F;IAC9F,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AA4GD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,eAAe,EAC3B,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,eAAe,EACxB,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,cAAc,CAAC,CAmCzB"}

package/dist/pairwise.js

@@ -0,0 +1,118 @@
+import { judgePrompt, parseJudgeOutput, sumUsage, untilAborted, usageOf, } from './metrics/llm-judge.js';
+import { conversationBlock, evidenceBlock, tagged, wrapped } from './metrics/judge-prompt.js';
+/** The instructions of a pairwise verdict */
+function pairwiseInstructions(criteria) {
+    const basis = criteria
+        ? `on these criteria:\n${criteria}`
+        : 'overall: which is more correct, relevant, complete and clear';
+    return `You are comparing two answers to the same request to decide which one is better ${basis}\n\nThe order of the answers says nothing about their quality, and a longer answer is better only for what its length adds.\n\nScore 0 when <answer_a> is better, 1 when <answer_b> is better, 0.5 when neither is better.\n\nRespond with JSON: {"score": <0, 0.5 or 1>, "reasoning": "<explanation>"}`;
+}
+/** The case and the two answers, A then B. A multi-turn case shows each side's whole conversation */
+function pairwiseMessage(a, b) {
+    const evidence = evidenceBlock(a, false);
+    const multiTurn = (a.case.turns?.length ?? 0) > 0;
+    const shared = [
+        ...(multiTurn
+            ? ['Each answer is a whole conversation, the user messages being the same in both.']
+            : [tagged('input', a.case.input)]),
+        ...(evidence === undefined ? [] : [evidence]),
+        ...(a.case.expected ? [tagged('expected', a.case.expected)] : []),
+    ];
+    const answer = (tag, result) => multiTurn ? wrapped(tag, conversationBlock(result)) : tagged(tag, result.output);
+    return [...shared, answer('answer_a', a), answer('answer_b', b)].join('\n');
+}
+const other = (side) => side === 'baseline' ? 'challenger' : 'baseline';
+async function ask(judge, instructions, first, a, b, signal) {
+    try {
+        const answered = await untilAborted(judge.cogitator.run({
+            input: judgePrompt(instructions, pairwiseMessage(a, b)),
+            ...(signal && { signal }),
+        }), signal);
+        const usage = usageOf(answered.usage);
+        const verdict = parseJudgeOutput(answered.output);
+        if (!verdict)
+            return { error: 'could not parse judge response', ...(usage && { usage }) };
+        const score = verdict.outOf === undefined ? verdict.score : verdict.score / verdict.outOf;
+        const preferred = score < 0.5 ? first : score > 0.5 ? other(first) : 'tie';
+        return {
+            verdict: {
+                first,
+                preferred,
+                ...(verdict.reasoning !== undefined && { reasoning: verdict.reasoning }),
+            },
+            ...(usage && { usage }),
+        };
+    }
+    catch (error) {
+        return { error: error instanceof Error ? error.message : String(error) };
+    }
+}
+/**
+ * A case judged both ways round: the baseline first, then the challenger first. Agreeing
+ * verdicts give the case to that side, disagreeing ones or a tie make it a tie
+ */
+async function judgeCase(judge, instructions, index, baseline, challenger, signal) {
+    const input = baseline.case.input;
+    if (baseline.error !== undefined || challenger.error !== undefined) {
+        const errored = baseline.error !== undefined ? 'baseline' : 'challenger';
+        return { result: { index, input, winner: other(errored), verdicts: [], errored } };
+    }
+    const asked = await Promise.all([
+        ask(judge, instructions, 'baseline', baseline, challenger, signal),
+        ask(judge, instructions, 'challenger', challenger, baseline, signal),
+    ]);
+    const usage = sumUsage(asked.map((one) => one.usage));
+    const verdicts = asked.flatMap((one) => (one.verdict ? [one.verdict] : []));
+    const errors = asked.flatMap((one) => (one.error === undefined ? [] : [one.error]));
+    const [first, second] = verdicts;
+    const winner = errors.length === 0 && first.preferred === second.preferred ? first.preferred : 'tie';
+    return {
+        result: {
+            index,
+            input,
+            winner,
+            verdicts,
+            ...(errors.length > 0 && { error: errors.join(', ') }),
+        },
+        ...(usage && { usage }),
+    };
+}
+/**
+ * Asks the judge which side answered each case better, see `EvalComparisonOptions.pairwise`.
+ * Cases both sides errored on are left out
+ */
+export async function judgePairwise(baseline, challenger, judge, options, concurrency, signal) {
+    const instructions = pairwiseInstructions(options.criteria);
+    const pairs = baseline.results.flatMap((b, index) => {
+        const c = challenger.results[index];
+        return c && (b.error === undefined || c.error === undefined) ? [{ index, b, c }] : [];
+    });
+    const judged = new Array(pairs.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < pairs.length && !signal?.aborted) {
+            const at = next++;
+            const { index, b, c } = pairs[at];
+            judged[at] = await judgeCase(judge, instructions, index, b, c, signal);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, pairs.length) }, worker));
+    signal?.throwIfAborted();
+    const perCase = judged.map((one) => one.result);
+    const wins = {
+        baseline: perCase.filter((one) => one.winner === 'baseline').length,
+        challenger: perCase.filter((one) => one.winner === 'challenger').length,
+    };
+    const decided = wins.baseline + wins.challenger;
+    return {
+        name: options.name ?? 'pairwise',
+        wins,
+        ties: perCase.length - decided,
+        cases: perCase.length,
+        winRate: decided === 0 ? Number.NaN : wins.challenger / decided,
+        judgeErrors: perCase.filter((one) => one.error !== undefined).length,
+        cost: judged.reduce((sum, one) => sum + (one.usage?.cost ?? 0), 0),
+        perCase,
+    };
+}
+//# sourceMappingURL=pairwise.js.map

package/dist/pairwise.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.js","sourceRoot":"","sources":["../src/pairwise.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,YAAY,EACZ,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAkD3F,6CAA6C;AAC7C,SAAS,oBAAoB,CAAC,QAA4B;IACxD,MAAM,KAAK,GAAG,QAAQ;QACpB,CAAC,CAAC,uBAAuB,QAAQ,EAAE;QACnC,CAAC,CAAC,8DAA8D,CAAC;IACnE,OAAO,mFAAmF,KAAK,ySAAyS,CAAC;AAC3Y,CAAC;AAED,qGAAqG;AACrG,SAAS,eAAe,CAAC,CAAmB,EAAE,CAAmB;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,SAAS;YACX,CAAC,CAAC,CAAC,gFAAgF,CAAC;YACpF,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACpC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC7C,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAClE,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,MAAwB,EAAE,EAAE,CACvD,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC;AAQD,MAAM,KAAK,GAAG,CAAC,IAAkB,EAAgB,EAAE,CACjD,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;AAElD,KAAK,UAAU,GAAG,CAChB,KAAmB,EACnB,YAAoB,EACpB,KAAmB,EACnB,CAAmB,EACnB,CAAmB,EACnB,MAA+B;IAE/B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,YAAY,CACjC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC;YAClB,KAAK,EAAE,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvD,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC;SAC1B,CAAC,EACF,MAAM,CACP,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC1F,MAAM,SAAS,GAAoB,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5F,OAAO;YACL,OAAO,EAAE;gBACP,KAAK;gBACL,SAAS;gBACT,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;aACzE;YACD,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3E,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,SAAS,CACtB,KAAmB,EACnB,YAAoB,EACpB,KAAa,EACb,QAA0B,EAC1B,UAA4B,EAC5B,MAA+B;IAE/B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;IAClC,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,OAAO,GAAiB,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;QACvF,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC9B,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC;QAClE,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;KACrE,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACpF,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,QAAQ,CAAC;IACjC,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IACxF,OAAO;QACL,MAAM,EAAE;YACN,KAAK;YACL,KAAK;YACL,MAAM;YACN,QAAQ;YACR,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;SACvD;QACD,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAyB,EACzB,UAA2B,EAC3B,KAAmB,EACnB,OAAwB,EACxB,WAAmB,EACnB,MAAoB;IAEpB,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QAClD,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,KAAK,CAAgD,KAAK,CAAC,MAAM,CAAC,CAAC;IACtF,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACzE,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACvF,MAAM,EAAE,cAAc,EAAE,CAAC;IAEzB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG;QACX,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM;QACnE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,MAAM;KACxE,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;IAChD,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,UAAU;QAChC,IAAI;QACJ,IAAI,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO;QAC9B,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,OAAO,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO;QAC/D,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;QACpE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/reporters/comparison.d.ts

@@ -0,0 +1,24 @@
+import type { ComparisonResult } from '../eval-comparison.js';
+import type { RunComparison } from '../runs/compare.js';
+import type { ReporterOptions } from './index.js';
+/** What `reportComparison()` renders: an `EvalComparison` result or a `compareRuns()` result */
+export type ReportableComparison = ComparisonResult | RunComparison;
+export type ComparisonReporterType = 'markdown' | 'html' | 'console';
+/**
+ * A compact Markdown summary of a comparison for a pull request comment: the winner, a table of
+ * the metrics with means, difference, interval, p-values and significance, the cases that
+ * regressed and improved, and the warnings. Every value is escaped
+ */
+export declare function comparisonMarkdown(comparison: ReportableComparison): string;
+/**
+ * A self-contained HTML page of a comparison: inline CSS for light and dark color schemes, no
+ * scripts, no external assets, and every value escaped
+ */
+export declare function comparisonHtml(comparison: ReportableComparison): string;
+/**
+ * Renders an `EvalComparison` result or a `compareRuns()` result: `console` prints it, `markdown`
+ * writes a compact summary for a pull request comment to `<path>.md`, and `html` a
+ * self-contained page to `<path>.html`. The default path is `eval-comparison`
+ */
+export declare function reportComparison(comparison: ReportableComparison, type: ComparisonReporterType | ComparisonReporterType[], options?: ReporterOptions): void;
+//# sourceMappingURL=comparison.d.ts.map

package/dist/reporters/comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison.d.ts","sourceRoot":"","sources":["../../src/reporters/comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAU,MAAM,iBAAiB,CAAC;AAK7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG/C,gGAAgG;AAChG,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAEpE,MAAM,MAAM,sBAAsB,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAsJrE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAqC3E;AAgBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAgFvE;AAoED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,oBAAoB,EAChC,IAAI,EAAE,sBAAsB,GAAG,sBAAsB,EAAE,EACvD,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAuBN"}

package/dist/reporters/comparison.js

@@ -0,0 +1,290 @@
+import { writeFileSync } from 'node:fs';
+import { caseChanges } from '../comparison-stats.js';
+import { cell, num, truncateInput } from './format.js';
+import { prepare, reportPath } from './index.js';
+import { escapeHtml, htmlDocument, htmlTable, htmlTiles } from './html-format.js';
+/** Most case changes the Markdown summary lists per metric and direction */
+const MARKDOWN_MAX_CHANGES = 10;
+function isRunComparison(comparison) {
+    return !('summary' in comparison);
+}
+function runLabel(ref) {
+    return ref.name === undefined ? ref.runId : `${ref.name} (${ref.runId})`;
+}
+function viewOf(comparison) {
+    if (isRunComparison(comparison)) {
+        return {
+            title: 'Run comparison',
+            before: 'base',
+            after: 'head',
+            subtitle: `base ${runLabel(comparison.base)}, head ${runLabel(comparison.head)}`,
+            winner: comparison.winner,
+            metrics: Object.entries(comparison.metrics).map(([name, m]) => ({
+                name,
+                before: m.base,
+                after: m.head,
+                n: m.n,
+                difference: m.difference,
+                ci: m.ci,
+                test: m.test,
+                pValue: m.pValue,
+                adjustedPValue: m.adjustedPValue,
+                significant: m.significant,
+                winner: m.winner,
+                improved: m.improved.map((c) => ({ ...c, before: c.base, after: c.head })),
+                regressed: m.regressed.map((c) => ({ ...c, before: c.base, after: c.head })),
+            })),
+            cases: {
+                matched: comparison.matched,
+                added: comparison.added.map((c) => c.input),
+                removed: comparison.removed.map((c) => c.input),
+            },
+            warnings: comparison.warnings,
+        };
+    }
+    const { baseline, challenger, summary } = comparison;
+    const pairs = baseline.results.map((r, at) => [r, challenger.results[at]]);
+    const inputOf = (index) => baseline.results[index].case.input;
+    return {
+        title: 'A/B comparison',
+        before: 'baseline',
+        after: 'challenger',
+        winner: summary.winner,
+        metrics: Object.entries(summary.metrics).map(([name, m]) => ({
+            name,
+            before: m.baseline,
+            after: m.challenger,
+            n: m.n,
+            difference: m.difference,
+            ci: m.ci,
+            test: m.test,
+            pValue: m.pValue,
+            adjustedPValue: m.adjustedPValue,
+            significant: m.significant,
+            winner: m.winner,
+            ...caseChanges(pairs, name, inputOf),
+        })),
+        warnings: [],
+    };
+}
+function signed(n) {
+    return n > 0 ? `+${num(n)}` : num(n);
+}
+function ciText(ci) {
+    return `[${num(ci.low)}, ${num(ci.high)}]`;
+}
+function pText(p) {
+    return p < 0.0001 ? '<0.0001' : num(p);
+}
+function changeLines(view, kind, heading) {
+    const rows = view.metrics.flatMap((m) => {
+        const changes = m[kind];
+        const shown = changes
+            .slice(0, MARKDOWN_MAX_CHANGES)
+            .map((c) => `| ${cell(m.name)} | ${cell(c.input)} | ${num(c.before)} | ${num(c.after)} | ${signed(c.delta)} |`);
+        const more = changes.length > MARKDOWN_MAX_CHANGES
+            ? [`| ${cell(m.name)} | and ${changes.length - MARKDOWN_MAX_CHANGES} more | | | |`]
+            : [];
+        return [...shown, ...more];
+    });
+    if (rows.length === 0)
+        return [];
+    return [
+        `**${heading}**`,
+        '',
+        `| Metric | Case | ${view.before} | ${view.after} | Change |`,
+        '| --- | --- | ---: | ---: | ---: |',
+        ...rows,
+        '',
+    ];
+}
+/**
+ * A compact Markdown summary of a comparison for a pull request comment: the winner, a table of
+ * the metrics with means, difference, interval, p-values and significance, the cases that
+ * regressed and improved, and the warnings. Every value is escaped
+ */
+export function comparisonMarkdown(comparison) {
+    const view = viewOf(comparison);
+    const lines = [`### ${view.title}`, ''];
+    if (view.subtitle)
+        lines.push(cell(view.subtitle), '');
+    const totals = [`winner: **${view.winner}**`];
+    if (view.cases) {
+        totals.push(`**${view.cases.matched}** cases matched`, `**${view.cases.added.length}** added`, `**${view.cases.removed.length}** removed`);
+    }
+    lines.push(totals.join(' | '), '');
+    if (view.warnings.length > 0) {
+        lines.push('**Warnings**', '', ...view.warnings.map((w) => `- ${cell(w)}`), '');
+    }
+    if (view.metrics.length > 0) {
+        lines.push(`| Metric | ${view.before} | ${view.after} | Difference | CI | p | Adjusted p | Significant | Winner |`, '| --- | ---: | ---: | ---: | --- | ---: | ---: | --- | --- |');
+        for (const m of view.metrics) {
+            lines.push(`| ${cell(m.name)} | ${num(m.before)} | ${num(m.after)} | ${signed(m.difference)} | ${ciText(m.ci)} | ${pText(m.pValue)} | ${pText(m.adjustedPValue)} | ${m.significant ? '**yes**' : 'no'} | ${m.winner} |`);
+        }
+        lines.push('');
+    }
+    else {
+        lines.push('No metric to compare.', '');
+    }
+    lines.push(...changeLines(view, 'regressed', 'Regressions'));
+    lines.push(...changeLines(view, 'improved', 'Improvements'));
+    return lines.join('\n');
+}
+function changesTable(view, kind) {
+    const rows = view.metrics.flatMap((m) => m[kind].map((c) => [
+        `<code>${escapeHtml(m.name)}</code>`,
+        escapeHtml(c.input),
+        escapeHtml(num(c.before)),
+        escapeHtml(num(c.after)),
+        `<span class="${c.delta < 0 ? 'fail' : 'pass'}">${escapeHtml(signed(c.delta))}</span>`,
+    ]));
+    if (rows.length === 0)
+        return undefined;
+    return htmlTable(['Metric', 'Case', view.before, view.after, 'Change'], rows, new Set([2, 3, 4]));
+}
+/**
+ * A self-contained HTML page of a comparison: inline CSS for light and dark color schemes, no
+ * scripts, no external assets, and every value escaped
+ */
+export function comparisonHtml(comparison) {
+    const view = viewOf(comparison);
+    const sections = [`<h1>${escapeHtml(view.title)}</h1>`];
+    if (view.subtitle)
+        sections.push(`<p class="sub">${escapeHtml(view.subtitle)}</p>`);
+    sections.push(htmlTiles([
+        [
+            'winner',
+            `<span class="${view.winner === 'tie' ? 'muted' : 'pass'}">${escapeHtml(view.winner)}</span>`,
+        ],
+        ...(view.cases
+            ? [
+                ['cases matched', escapeHtml(String(view.cases.matched))],
+                ['added', escapeHtml(String(view.cases.added.length))],
+                ['removed', escapeHtml(String(view.cases.removed.length))],
+            ]
+            : []),
+    ]));
+    if (view.warnings.length > 0) {
+        sections.push('<h2>Warnings</h2>', `<ul>${view.warnings.map((w) => `<li class="warn">${escapeHtml(w)}</li>`).join('')}</ul>`);
+    }
+    sections.push('<h2>Metrics</h2>', view.metrics.length === 0
+        ? '<p class="muted">No metric to compare.</p>'
+        : htmlTable([
+            'Metric',
+            view.before,
+            view.after,
+            'Cases',
+            'Difference',
+            'CI',
+            'Test',
+            'p',
+            'Adjusted p',
+            'Significant',
+            'Winner',
+        ], view.metrics.map((m) => [
+            `<code>${escapeHtml(m.name)}</code>`,
+            escapeHtml(num(m.before)),
+            escapeHtml(num(m.after)),
+            escapeHtml(String(m.n)),
+            escapeHtml(signed(m.difference)),
+            escapeHtml(ciText(m.ci)),
+            escapeHtml(m.test),
+            escapeHtml(pText(m.pValue)),
+            escapeHtml(pText(m.adjustedPValue)),
+            m.significant ? '<span class="pass">yes</span>' : '<span class="muted">no</span>',
+            escapeHtml(m.winner),
+        ]), new Set([1, 2, 3, 4, 7, 8])));
+    const regressed = changesTable(view, 'regressed');
+    if (regressed)
+        sections.push('<h2>Regressions</h2>', regressed);
+    const improved = changesTable(view, 'improved');
+    if (improved)
+        sections.push('<h2>Improvements</h2>', improved);
+    if (view.cases && (view.cases.added.length > 0 || view.cases.removed.length > 0)) {
+        const list = (inputs) => `<ul>${inputs.map((input) => `<li>${escapeHtml(input)}</li>`).join('')}</ul>`;
+        if (view.cases.added.length > 0) {
+            sections.push(`<h2>Cases only in the ${view.after} run</h2>`, list(view.cases.added));
+        }
+        if (view.cases.removed.length > 0) {
+            sections.push(`<h2>Cases only in the ${view.before} run</h2>`, list(view.cases.removed));
+        }
+    }
+    return htmlDocument(view.title, sections.join('\n'));
+}
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const YELLOW = '\x1b[33m';
+const BOLD = '\x1b[1m';
+const DIM = '\x1b[2m';
+const RESET = '\x1b[0m';
+function pad(text, width) {
+    return text.length >= width ? `${text} ` : text + ' '.repeat(width - text.length);
+}
+function printConsole(comparison) {
+    const view = viewOf(comparison);
+    console.log(`\n${BOLD}${view.title}${RESET}${view.subtitle ? ` ${DIM}${view.subtitle}${RESET}` : ''}`);
+    const cases = view.cases
+        ? ` | ${view.cases.matched} cases matched, ${view.cases.added.length} added, ${view.cases.removed.length} removed`
+        : '';
+    console.log(`Winner: ${BOLD}${view.winner}${RESET}${cases}`);
+    for (const warning of view.warnings)
+        console.log(`${YELLOW}warning${RESET} ${warning}`);
+    if (view.metrics.length > 0) {
+        const width = Math.max(14, ...view.metrics.map((m) => m.name.length + 2));
+        const header = pad('Metric', width) +
+            pad(view.before, 12) +
+            pad(view.after, 12) +
+            pad('Diff', 10) +
+            pad('CI', 22) +
+            pad('p', 10) +
+            pad('Adj. p', 10) +
+            'Winner';
+        console.log(`\n${BOLD}${header}${RESET}`);
+        console.log(DIM + '─'.repeat(header.length) + RESET);
+        for (const m of view.metrics) {
+            const color = !m.significant ? '' : m.winner === view.after ? GREEN : RED;
+            console.log(pad(m.name, width) +
+                pad(num(m.before), 12) +
+                pad(num(m.after), 12) +
+                pad(signed(m.difference), 10) +
+                pad(ciText(m.ci), 22) +
+                pad(pText(m.pValue), 10) +
+                pad(pText(m.adjustedPValue), 10) +
+                `${color}${m.winner}${m.significant ? ' *' : ''}${color ? RESET : ''}`);
+        }
+    }
+    for (const [kind, heading, color] of [
+        ['regressed', 'Regressions', RED],
+        ['improved', 'Improvements', GREEN],
+    ]) {
+        const changes = view.metrics.flatMap((m) => m[kind].map((c) => ({ metric: m.name, ...c })));
+        if (changes.length === 0)
+            continue;
+        console.log(`\n${BOLD}${heading}${RESET}`);
+        for (const c of changes) {
+            console.log(`  ${color}${signed(c.delta)}${RESET} ${c.metric} ${DIM}${num(c.before)} -> ${num(c.after)}${RESET} ${truncateInput(c.input)}`);
+        }
+    }
+}
+/**
+ * Renders an `EvalComparison` result or a `compareRuns()` result: `console` prints it, `markdown`
+ * writes a compact summary for a pull request comment to `<path>.md`, and `html` a
+ * self-contained page to `<path>.html`. The default path is `eval-comparison`
+ */
+export function reportComparison(comparison, type, options) {
+    const types = Array.isArray(type) ? type : [type];
+    for (const t of types) {
+        switch (t) {
+            case 'console':
+                printConsole(comparison);
+                break;
+            case 'markdown':
+                writeFileSync(prepare(reportPath(options?.path, 'md', 'eval-comparison')), comparisonMarkdown(comparison), 'utf-8');
+                break;
+            case 'html':
+                writeFileSync(prepare(reportPath(options?.path, 'html', 'eval-comparison')), comparisonHtml(comparison), 'utf-8');
+                break;
+        }
+    }
+}
+//# sourceMappingURL=comparison.js.map

package/dist/reporters/comparison.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison.js","sourceRoot":"","sources":["../../src/reporters/comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAIxC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AA4C/E,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,SAAS,eAAe,CAAC,UAAgC;IACvD,OAAO,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,OAAO,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC;AAC3E,CAAC;AAED,SAAS,MAAM,CAAC,UAAgC;IAC9C,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,QAAQ,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YAChF,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9D,IAAI;gBACJ,MAAM,EAAE,CAAC,CAAC,IAAI;gBACd,KAAK,EAAE,CAAC,CAAC,IAAI;gBACb,CAAC,EAAE,CAAC,CAAC,CAAC;gBACN,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,cAAc,EAAE,CAAC,CAAC,cAAc;gBAChC,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1E,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aAC7E,CAAC,CAAC;YACH,KAAK,EAAE;gBACL,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3C,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;aAChD;YACD,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;IACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;IACtE,OAAO;QACL,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI;YACJ,MAAM,EAAE,CAAC,CAAC,QAAQ;YAClB,KAAK,EAAE,CAAC,CAAC,UAAU;YACnB,CAAC,EAAE,CAAC,CAAC,CAAC;YACN,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC;SACrC,CAAC,CAAC;QACH,QAAQ,EAAE,EAAE;KACb,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,MAAM,CAAC,EAAiC;IAC/C,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;AAC7C,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAClB,IAAoB,EACpB,IAA8B,EAC9B,OAAe;IAEf,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO;aAClB,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;aAC9B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CACrG,CAAC;QACJ,MAAM,IAAI,GACR,OAAO,CAAC,MAAM,GAAG,oBAAoB;YACnC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,GAAG,oBAAoB,eAAe,CAAC;YACnF,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO;QACL,KAAK,OAAO,IAAI;QAChB,EAAE;QACF,qBAAqB,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,aAAa;QAC7D,oCAAoC;QACpC,GAAG,IAAI;QACP,EAAE;KACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAgC;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,CAAC,aAAa,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CACT,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,kBAAkB,EACzC,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,UAAU,EACtC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,YAAY,CAC3C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CACR,cAAc,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,8DAA8D,EACvG,8DAA8D,CAC/D,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,IAAI,CAC7M,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IAC7D,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;IAC7D,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,IAAoB,EAAE,IAA8B;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjB,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;QACpC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QACnB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxB,gBAAgB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS;KACvF,CAAC,CACH,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,OAAO,SAAS,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAgC;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxD,IAAI,IAAI,CAAC,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEpF,QAAQ,CAAC,IAAI,CACX,SAAS,CAAC;QACR;YACE,QAAQ;YACR,gBAAgB,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS;SAC9F;QACD,GAAG,CAAC,IAAI,CAAC,KAAK;YACZ,CAAC,CAAE;gBACC,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACzD,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtD,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;aACjD;YACb,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CACH,CAAC;IAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CACX,mBAAmB,EACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAC1F,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,IAAI,CACX,kBAAkB,EAClB,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,4CAA4C;QAC9C,CAAC,CAAC,SAAS,CACP;YACE,QAAQ;YACR,IAAI,CAAC,MAAM;YACX,IAAI,CAAC,KAAK;YACV,OAAO;YACP,YAAY;YACZ,IAAI;YACJ,MAAM;YACN,GAAG;YACH,YAAY;YACZ,aAAa;YACb,QAAQ;SACT,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACtB,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;YACpC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACzB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAChC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxB,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YAClB,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC3B,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACnC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,+BAA+B;YACjF,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;SACrB,CAAC,EACF,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAC5B,CACN,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAClD,IAAI,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAChD,IAAI,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;IAE/D,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CAChC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;QAChF,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,KAAK,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,MAAM,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,IAAY,EAAE,KAAa;IACtC,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,YAAY,CAAC,UAAgC;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1F,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK;QACtB,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,mBAAmB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,WAAW,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,UAAU;QAClH,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;IAE7D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,KAAK,IAAI,OAAO,EAAE,CAAC,CAAC;IAExF,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACnB,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YACb,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,QAAQ,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1E,OAAO,CAAC,GAAG,CACT,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC;gBAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACtB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;gBAC7B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACxB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC;gBAChC,GAAG,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACzE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI;QACnC,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,CAAC;QACjC,CAAC,UAAU,EAAE,cAAc,EAAE,KAAK,CAAC;KAC3B,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACnC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAC/H,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,UAAgC,EAChC,IAAuD,EACvD,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM;YACR,KAAK,UAAU;gBACb,aAAa,CACX,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC,EAC3D,kBAAkB,CAAC,UAAU,CAAC,EAC9B,OAAO,CACR,CAAC;gBACF,MAAM;YACR,KAAK,MAAM;gBACT,aAAa,CACX,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,EAC7D,cAAc,CAAC,UAAU,CAAC,EAC1B,OAAO,CACR,CAAC;gBACF,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC"}