@cogitator-ai/evals 0.12.0 → 0.13.1

package/dist/index.js

@@ -21,6 +21,8 @@ export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
 export { maxErrorRate } from './assertions/index.js';
 export { report } from './reporters/index.js';
+export { reportComparison, comparisonMarkdown, comparisonHtml } from './reporters/comparison.js';
+export { loadRun, caseKey, RunStore, compareRuns, EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION, } from './runs/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export { mcnemarsTest, mcnemarExact } from './stats/mcnemar.js';
 export { wilcoxonSignedRank } from './stats/wilcoxon.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAkBnD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAe7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAkBnD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAe7B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAkB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAG9F,OAAO,EACL,OAAO,EACP,OAAO,EACP,QAAQ,EACR,WAAW,EACX,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,MAAM,QAAQ,CAAC;AAoBhB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/types.d.ts

@@ -39,11 +39,26 @@ export interface Passage {
     score?: number;
     source?: string;
 }
+/** The versioned instructions an agent run used, as its `RunResult.prompt` reports them */
+export interface EvalPrompt {
+    /** The agent's `id` when it was set explicitly, else its `name` */
+    key: string;
+    versionId?: string;
+    version?: number;
+    abTest?: {
+        id: string;
+        variant: 'control' | 'treatment';
+    };
+}
 /** One user turn of a multi-turn case and the answer to it */
 export interface EvalTurnResult {
     input: string;
     output: string;
     toolCalls?: readonly EvalToolCall[];
+    /** The trace of the agent run that answered the turn */
+    traceId?: string;
+    /** The versioned instructions the agent run of the turn used */
+    prompt?: EvalPrompt;
     /** Set on the turn that failed, the last one the case ran */
     error?: string;
 }
@@ -74,6 +89,16 @@ export interface EvalCaseResult {
     retrieved?: readonly Passage[];
     /** Each turn of a case with `turns`: `input` first, then the further turns */
     turns?: readonly EvalTurnResult[];
+    /**
+     * The trace of the agent run, from its `RunResult.trace.traceId`, to find the run in a trace
+     * viewer. With `turns`, the last turn's. A `fn` target has none
+     */
+    traceId?: string;
+    /**
+     * The versioned instructions the agent run used, from its `RunResult.prompt`. With `turns`, the
+     * last turn's. Set when the Cogitator versions instructions (`cogitator.prompts`)
+     */
+    prompt?: EvalPrompt;
     /**
      * Every run of the case when the suite runs several `trials`, each with its own scores. The
      * case result itself is the first of them
@@ -85,7 +110,7 @@ export interface EvalCaseResult {
     truncated?: boolean;
     /** Tool calls used up the agent's `maxIterations` before it answered */
     iterationLimitReached?: boolean;
-    /** The run paused for tool approvals, so `output` holds no answer */
+    /** The run paused for tool approvals or questions of its tools, so `output` holds no answer */
     paused?: boolean;
 }
 export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {

package/dist/metrics/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,2FAA2F;AAC3F,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,SAAS,GAAG,WAAW,CAAA;KAAE,CAAC;CAC3D;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,+FAA+F;IAC/F,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}

package/dist/metrics/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA4HA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAmJA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}

package/dist/reporters/comparison.d.ts

@@ -0,0 +1,24 @@
+import type { ComparisonResult } from '../eval-comparison.js';
+import type { RunComparison } from '../runs/compare.js';
+import type { ReporterOptions } from './index.js';
+/** What `reportComparison()` renders: an `EvalComparison` result or a `compareRuns()` result */
+export type ReportableComparison = ComparisonResult | RunComparison;
+export type ComparisonReporterType = 'markdown' | 'html' | 'console';
+/**
+ * A compact Markdown summary of a comparison for a pull request comment: the winner, a table of
+ * the metrics with means, difference, interval, p-values and significance, the cases that
+ * regressed and improved, and the warnings. Every value is escaped
+ */
+export declare function comparisonMarkdown(comparison: ReportableComparison): string;
+/**
+ * A self-contained HTML page of a comparison: inline CSS for light and dark color schemes, no
+ * scripts, no external assets, and every value escaped
+ */
+export declare function comparisonHtml(comparison: ReportableComparison): string;
+/**
+ * Renders an `EvalComparison` result or a `compareRuns()` result: `console` prints it, `markdown`
+ * writes a compact summary for a pull request comment to `<path>.md`, and `html` a
+ * self-contained page to `<path>.html`. The default path is `eval-comparison`
+ */
+export declare function reportComparison(comparison: ReportableComparison, type: ComparisonReporterType | ComparisonReporterType[], options?: ReporterOptions): void;
+//# sourceMappingURL=comparison.d.ts.map

package/dist/reporters/comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison.d.ts","sourceRoot":"","sources":["../../src/reporters/comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAU,MAAM,iBAAiB,CAAC;AAK7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG/C,gGAAgG;AAChG,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAEpE,MAAM,MAAM,sBAAsB,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAsJrE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAqC3E;AAgBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAgFvE;AAoED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,oBAAoB,EAChC,IAAI,EAAE,sBAAsB,GAAG,sBAAsB,EAAE,EACvD,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAuBN"}

package/dist/reporters/comparison.js

@@ -0,0 +1,290 @@
+import { writeFileSync } from 'node:fs';
+import { caseChanges } from '../comparison-stats.js';
+import { cell, num, truncateInput } from './format.js';
+import { prepare, reportPath } from './index.js';
+import { escapeHtml, htmlDocument, htmlTable, htmlTiles } from './html-format.js';
+/** Most case changes the Markdown summary lists per metric and direction */
+const MARKDOWN_MAX_CHANGES = 10;
+function isRunComparison(comparison) {
+    return !('summary' in comparison);
+}
+function runLabel(ref) {
+    return ref.name === undefined ? ref.runId : `${ref.name} (${ref.runId})`;
+}
+function viewOf(comparison) {
+    if (isRunComparison(comparison)) {
+        return {
+            title: 'Run comparison',
+            before: 'base',
+            after: 'head',
+            subtitle: `base ${runLabel(comparison.base)}, head ${runLabel(comparison.head)}`,
+            winner: comparison.winner,
+            metrics: Object.entries(comparison.metrics).map(([name, m]) => ({
+                name,
+                before: m.base,
+                after: m.head,
+                n: m.n,
+                difference: m.difference,
+                ci: m.ci,
+                test: m.test,
+                pValue: m.pValue,
+                adjustedPValue: m.adjustedPValue,
+                significant: m.significant,
+                winner: m.winner,
+                improved: m.improved.map((c) => ({ ...c, before: c.base, after: c.head })),
+                regressed: m.regressed.map((c) => ({ ...c, before: c.base, after: c.head })),
+            })),
+            cases: {
+                matched: comparison.matched,
+                added: comparison.added.map((c) => c.input),
+                removed: comparison.removed.map((c) => c.input),
+            },
+            warnings: comparison.warnings,
+        };
+    }
+    const { baseline, challenger, summary } = comparison;
+    const pairs = baseline.results.map((r, at) => [r, challenger.results[at]]);
+    const inputOf = (index) => baseline.results[index].case.input;
+    return {
+        title: 'A/B comparison',
+        before: 'baseline',
+        after: 'challenger',
+        winner: summary.winner,
+        metrics: Object.entries(summary.metrics).map(([name, m]) => ({
+            name,
+            before: m.baseline,
+            after: m.challenger,
+            n: m.n,
+            difference: m.difference,
+            ci: m.ci,
+            test: m.test,
+            pValue: m.pValue,
+            adjustedPValue: m.adjustedPValue,
+            significant: m.significant,
+            winner: m.winner,
+            ...caseChanges(pairs, name, inputOf),
+        })),
+        warnings: [],
+    };
+}
+function signed(n) {
+    return n > 0 ? `+${num(n)}` : num(n);
+}
+function ciText(ci) {
+    return `[${num(ci.low)}, ${num(ci.high)}]`;
+}
+function pText(p) {
+    return p < 0.0001 ? '<0.0001' : num(p);
+}
+function changeLines(view, kind, heading) {
+    const rows = view.metrics.flatMap((m) => {
+        const changes = m[kind];
+        const shown = changes
+            .slice(0, MARKDOWN_MAX_CHANGES)
+            .map((c) => `| ${cell(m.name)} | ${cell(c.input)} | ${num(c.before)} | ${num(c.after)} | ${signed(c.delta)} |`);
+        const more = changes.length > MARKDOWN_MAX_CHANGES
+            ? [`| ${cell(m.name)} | and ${changes.length - MARKDOWN_MAX_CHANGES} more | | | |`]
+            : [];
+        return [...shown, ...more];
+    });
+    if (rows.length === 0)
+        return [];
+    return [
+        `**${heading}**`,
+        '',
+        `| Metric | Case | ${view.before} | ${view.after} | Change |`,
+        '| --- | --- | ---: | ---: | ---: |',
+        ...rows,
+        '',
+    ];
+}
+/**
+ * A compact Markdown summary of a comparison for a pull request comment: the winner, a table of
+ * the metrics with means, difference, interval, p-values and significance, the cases that
+ * regressed and improved, and the warnings. Every value is escaped
+ */
+export function comparisonMarkdown(comparison) {
+    const view = viewOf(comparison);
+    const lines = [`### ${view.title}`, ''];
+    if (view.subtitle)
+        lines.push(cell(view.subtitle), '');
+    const totals = [`winner: **${view.winner}**`];
+    if (view.cases) {
+        totals.push(`**${view.cases.matched}** cases matched`, `**${view.cases.added.length}** added`, `**${view.cases.removed.length}** removed`);
+    }
+    lines.push(totals.join(' | '), '');
+    if (view.warnings.length > 0) {
+        lines.push('**Warnings**', '', ...view.warnings.map((w) => `- ${cell(w)}`), '');
+    }
+    if (view.metrics.length > 0) {
+        lines.push(`| Metric | ${view.before} | ${view.after} | Difference | CI | p | Adjusted p | Significant | Winner |`, '| --- | ---: | ---: | ---: | --- | ---: | ---: | --- | --- |');
+        for (const m of view.metrics) {
+            lines.push(`| ${cell(m.name)} | ${num(m.before)} | ${num(m.after)} | ${signed(m.difference)} | ${ciText(m.ci)} | ${pText(m.pValue)} | ${pText(m.adjustedPValue)} | ${m.significant ? '**yes**' : 'no'} | ${m.winner} |`);
+        }
+        lines.push('');
+    }
+    else {
+        lines.push('No metric to compare.', '');
+    }
+    lines.push(...changeLines(view, 'regressed', 'Regressions'));
+    lines.push(...changeLines(view, 'improved', 'Improvements'));
+    return lines.join('\n');
+}
+function changesTable(view, kind) {
+    const rows = view.metrics.flatMap((m) => m[kind].map((c) => [
+        `<code>${escapeHtml(m.name)}</code>`,
+        escapeHtml(c.input),
+        escapeHtml(num(c.before)),
+        escapeHtml(num(c.after)),
+        `<span class="${c.delta < 0 ? 'fail' : 'pass'}">${escapeHtml(signed(c.delta))}</span>`,
+    ]));
+    if (rows.length === 0)
+        return undefined;
+    return htmlTable(['Metric', 'Case', view.before, view.after, 'Change'], rows, new Set([2, 3, 4]));
+}
+/**
+ * A self-contained HTML page of a comparison: inline CSS for light and dark color schemes, no
+ * scripts, no external assets, and every value escaped
+ */
+export function comparisonHtml(comparison) {
+    const view = viewOf(comparison);
+    const sections = [`<h1>${escapeHtml(view.title)}</h1>`];
+    if (view.subtitle)
+        sections.push(`<p class="sub">${escapeHtml(view.subtitle)}</p>`);
+    sections.push(htmlTiles([
+        [
+            'winner',
+            `<span class="${view.winner === 'tie' ? 'muted' : 'pass'}">${escapeHtml(view.winner)}</span>`,
+        ],
+        ...(view.cases
+            ? [
+                ['cases matched', escapeHtml(String(view.cases.matched))],
+                ['added', escapeHtml(String(view.cases.added.length))],
+                ['removed', escapeHtml(String(view.cases.removed.length))],
+            ]
+            : []),
+    ]));
+    if (view.warnings.length > 0) {
+        sections.push('<h2>Warnings</h2>', `<ul>${view.warnings.map((w) => `<li class="warn">${escapeHtml(w)}</li>`).join('')}</ul>`);
+    }
+    sections.push('<h2>Metrics</h2>', view.metrics.length === 0
+        ? '<p class="muted">No metric to compare.</p>'
+        : htmlTable([
+            'Metric',
+            view.before,
+            view.after,
+            'Cases',
+            'Difference',
+            'CI',
+            'Test',
+            'p',
+            'Adjusted p',
+            'Significant',
+            'Winner',
+        ], view.metrics.map((m) => [
+            `<code>${escapeHtml(m.name)}</code>`,
+            escapeHtml(num(m.before)),
+            escapeHtml(num(m.after)),
+            escapeHtml(String(m.n)),
+            escapeHtml(signed(m.difference)),
+            escapeHtml(ciText(m.ci)),
+            escapeHtml(m.test),
+            escapeHtml(pText(m.pValue)),
+            escapeHtml(pText(m.adjustedPValue)),
+            m.significant ? '<span class="pass">yes</span>' : '<span class="muted">no</span>',
+            escapeHtml(m.winner),
+        ]), new Set([1, 2, 3, 4, 7, 8])));
+    const regressed = changesTable(view, 'regressed');
+    if (regressed)
+        sections.push('<h2>Regressions</h2>', regressed);
+    const improved = changesTable(view, 'improved');
+    if (improved)
+        sections.push('<h2>Improvements</h2>', improved);
+    if (view.cases && (view.cases.added.length > 0 || view.cases.removed.length > 0)) {
+        const list = (inputs) => `<ul>${inputs.map((input) => `<li>${escapeHtml(input)}</li>`).join('')}</ul>`;
+        if (view.cases.added.length > 0) {
+            sections.push(`<h2>Cases only in the ${view.after} run</h2>`, list(view.cases.added));
+        }
+        if (view.cases.removed.length > 0) {
+            sections.push(`<h2>Cases only in the ${view.before} run</h2>`, list(view.cases.removed));
+        }
+    }
+    return htmlDocument(view.title, sections.join('\n'));
+}
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const YELLOW = '\x1b[33m';
+const BOLD = '\x1b[1m';
+const DIM = '\x1b[2m';
+const RESET = '\x1b[0m';
+function pad(text, width) {
+    return text.length >= width ? `${text} ` : text + ' '.repeat(width - text.length);
+}
+function printConsole(comparison) {
+    const view = viewOf(comparison);
+    console.log(`\n${BOLD}${view.title}${RESET}${view.subtitle ? ` ${DIM}${view.subtitle}${RESET}` : ''}`);
+    const cases = view.cases
+        ? ` | ${view.cases.matched} cases matched, ${view.cases.added.length} added, ${view.cases.removed.length} removed`
+        : '';
+    console.log(`Winner: ${BOLD}${view.winner}${RESET}${cases}`);
+    for (const warning of view.warnings)
+        console.log(`${YELLOW}warning${RESET} ${warning}`);
+    if (view.metrics.length > 0) {
+        const width = Math.max(14, ...view.metrics.map((m) => m.name.length + 2));
+        const header = pad('Metric', width) +
+            pad(view.before, 12) +
+            pad(view.after, 12) +
+            pad('Diff', 10) +
+            pad('CI', 22) +
+            pad('p', 10) +
+            pad('Adj. p', 10) +
+            'Winner';
+        console.log(`\n${BOLD}${header}${RESET}`);
+        console.log(DIM + '─'.repeat(header.length) + RESET);
+        for (const m of view.metrics) {
+            const color = !m.significant ? '' : m.winner === view.after ? GREEN : RED;
+            console.log(pad(m.name, width) +
+                pad(num(m.before), 12) +
+                pad(num(m.after), 12) +
+                pad(signed(m.difference), 10) +
+                pad(ciText(m.ci), 22) +
+                pad(pText(m.pValue), 10) +
+                pad(pText(m.adjustedPValue), 10) +
+                `${color}${m.winner}${m.significant ? ' *' : ''}${color ? RESET : ''}`);
+        }
+    }
+    for (const [kind, heading, color] of [
+        ['regressed', 'Regressions', RED],
+        ['improved', 'Improvements', GREEN],
+    ]) {
+        const changes = view.metrics.flatMap((m) => m[kind].map((c) => ({ metric: m.name, ...c })));
+        if (changes.length === 0)
+            continue;
+        console.log(`\n${BOLD}${heading}${RESET}`);
+        for (const c of changes) {
+            console.log(`  ${color}${signed(c.delta)}${RESET} ${c.metric} ${DIM}${num(c.before)} -> ${num(c.after)}${RESET} ${truncateInput(c.input)}`);
+        }
+    }
+}
+/**
+ * Renders an `EvalComparison` result or a `compareRuns()` result: `console` prints it, `markdown`
+ * writes a compact summary for a pull request comment to `<path>.md`, and `html` a
+ * self-contained page to `<path>.html`. The default path is `eval-comparison`
+ */
+export function reportComparison(comparison, type, options) {
+    const types = Array.isArray(type) ? type : [type];
+    for (const t of types) {
+        switch (t) {
+            case 'console':
+                printConsole(comparison);
+                break;
+            case 'markdown':
+                writeFileSync(prepare(reportPath(options?.path, 'md', 'eval-comparison')), comparisonMarkdown(comparison), 'utf-8');
+                break;
+            case 'html':
+                writeFileSync(prepare(reportPath(options?.path, 'html', 'eval-comparison')), comparisonHtml(comparison), 'utf-8');
+                break;
+        }
+    }
+}
+//# sourceMappingURL=comparison.js.map

package/dist/reporters/comparison.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison.js","sourceRoot":"","sources":["../../src/reporters/comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAIxC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AA4C/E,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,SAAS,eAAe,CAAC,UAAgC;IACvD,OAAO,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,OAAO,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC;AAC3E,CAAC;AAED,SAAS,MAAM,CAAC,UAAgC;IAC9C,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,QAAQ,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YAChF,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9D,IAAI;gBACJ,MAAM,EAAE,CAAC,CAAC,IAAI;gBACd,KAAK,EAAE,CAAC,CAAC,IAAI;gBACb,CAAC,EAAE,CAAC,CAAC,CAAC;gBACN,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,cAAc,EAAE,CAAC,CAAC,cAAc;gBAChC,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1E,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aAC7E,CAAC,CAAC;YACH,KAAK,EAAE;gBACL,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3C,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;aAChD;YACD,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;IACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;IACtE,OAAO;QACL,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI;YACJ,MAAM,EAAE,CAAC,CAAC,QAAQ;YAClB,KAAK,EAAE,CAAC,CAAC,UAAU;YACnB,CAAC,EAAE,CAAC,CAAC,CAAC;YACN,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC;SACrC,CAAC,CAAC;QACH,QAAQ,EAAE,EAAE;KACb,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,MAAM,CAAC,EAAiC;IAC/C,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;AAC7C,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAClB,IAAoB,EACpB,IAA8B,EAC9B,OAAe;IAEf,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO;aAClB,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;aAC9B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CACrG,CAAC;QACJ,MAAM,IAAI,GACR,OAAO,CAAC,MAAM,GAAG,oBAAoB;YACnC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,GAAG,oBAAoB,eAAe,CAAC;YACnF,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO;QACL,KAAK,OAAO,IAAI;QAChB,EAAE;QACF,qBAAqB,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,aAAa;QAC7D,oCAAoC;QACpC,GAAG,IAAI;QACP,EAAE;KACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAgC;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,CAAC,aAAa,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CACT,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,kBAAkB,EACzC,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,UAAU,EACtC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,YAAY,CAC3C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CACR,cAAc,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,8DAA8D,EACvG,8DAA8D,CAC/D,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,IAAI,CAC7M,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IAC7D,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;IAC7D,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,IAAoB,EAAE,IAA8B;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjB,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;QACpC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QACnB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxB,gBAAgB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS;KACvF,CAAC,CACH,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,OAAO,SAAS,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAgC;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxD,IAAI,IAAI,CAAC,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEpF,QAAQ,CAAC,IAAI,CACX,SAAS,CAAC;QACR;YACE,QAAQ;YACR,gBAAgB,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS;SAC9F;QACD,GAAG,CAAC,IAAI,CAAC,KAAK;YACZ,CAAC,CAAE;gBACC,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACzD,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtD,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;aACjD;YACb,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CACH,CAAC;IAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CACX,mBAAmB,EACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAC1F,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,IAAI,CACX,kBAAkB,EAClB,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,4CAA4C;QAC9C,CAAC,CAAC,SAAS,CACP;YACE,QAAQ;YACR,IAAI,CAAC,MAAM;YACX,IAAI,CAAC,KAAK;YACV,OAAO;YACP,YAAY;YACZ,IAAI;YACJ,MAAM;YACN,GAAG;YACH,YAAY;YACZ,aAAa;YACb,QAAQ;SACT,EACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACtB,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;YACpC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACzB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAChC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxB,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YAClB,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC3B,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACnC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,+BAA+B;YACjF,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;SACrB,CAAC,EACF,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAC5B,CACN,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAClD,IAAI,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAChD,IAAI,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;IAE/D,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CAChC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;QAChF,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,KAAK,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,MAAM,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,IAAY,EAAE,KAAa;IACtC,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,YAAY,CAAC,UAAgC;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1F,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK;QACtB,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,mBAAmB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,WAAW,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,UAAU;QAClH,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;IAE7D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,KAAK,IAAI,OAAO,EAAE,CAAC,CAAC;IAExF,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACnB,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YACb,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,QAAQ,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1E,OAAO,CAAC,GAAG,CACT,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC;gBAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACtB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;gBAC7B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACxB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC;gBAChC,GAAG,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACzE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI;QACnC,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,CAAC;QACjC,CAAC,UAAU,EAAE,cAAc,EAAE,KAAK,CAAC;KAC3B,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACnC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAC/H,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,UAAgC,EAChC,IAAuD,EACvD,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM;YACR,KAAK,UAAU;gBACb,aAAa,CACX,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC,EAC3D,kBAAkB,CAAC,UAAU,CAAC,EAC9B,OAAO,CACR,CAAC;gBACF,MAAM;YACR,KAAK,MAAM;gBACT,aAAa,CACX,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,EAC7D,cAAc,CAAC,UAAU,CAAC,EAC1B,OAAO,CACR,CAAC;gBACF,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/reporters/format.d.ts

@@ -16,4 +16,25 @@ export declare function trialsPerCase(result: {
         trials?: readonly unknown[];
     }>;
 }): number;
+/**
+ * A Markdown table cell: any line break (a lone `\r` too) becomes `<br>` so the row holds, and
+ * `<` and `&` are escaped so no tag or comment in an answer hides the rest
+ */
+export declare function cell(value: string): string;
+/** A figure in a report: an integer as it is, any other number to 4 decimals */
+export declare function num(n: number): string;
+/** A case result as `caseProblems` reads it */
+export interface ProblemSource {
+    error?: string;
+    scores: ReadonlyArray<{
+        name: string;
+        error?: string;
+    }>;
+    trials?: readonly ProblemSource[];
+}
+/**
+ * What went wrong on a case: its error and each metric that failed on it. With several trials,
+ * those of every trial, each prefixed with its trial number
+ */
+export declare function caseProblems(result: ProblemSource): string[];
 //# sourceMappingURL=format.d.ts.map

package/dist/reporters/format.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,OAAO,EAAE,aAAa,CAAC;QAAE,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CACzD,GAAG,MAAM,CAET"}
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,OAAO,EAAE,aAAa,CAAC;QAAE,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CACzD,GAAG,MAAM,CAET;AAED;;;GAGG;AACH,wBAAgB,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAO1C;AAED,gFAAgF;AAChF,wBAAgB,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAErC;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxD,MAAM,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACnC;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,EAAE,CAY5D"}

package/dist/reporters/format.js

@@ -28,4 +28,35 @@ export function caseLabel(index, input) {
 export function trialsPerCase(result) {
     return result.results.reduce((most, r) => Math.max(most, r.trials?.length ?? 1), 1);
 }
+/**
+ * A Markdown table cell: any line break (a lone `\r` too) becomes `<br>` so the row holds, and
+ * `<` and `&` are escaped so no tag or comment in an answer hides the rest
+ */
+export function cell(value) {
+    return value
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/\\/g, '\\\\')
+        .replace(/\|/g, '\\|')
+        .replace(/\r\n|\r|\n/g, '<br>');
+}
+/** A figure in a report: an integer as it is, any other number to 4 decimals */
+export function num(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(4);
+}
+/**
+ * What went wrong on a case: its error and each metric that failed on it. With several trials,
+ * those of every trial, each prefixed with its trial number
+ */
+export function caseProblems(result) {
+    const several = result.trials !== undefined && result.trials.length > 1;
+    const runs = several ? result.trials : [result];
+    return runs.flatMap((run, at) => {
+        const prefix = several ? `trial ${at + 1}: ` : '';
+        return [
+            ...(run.error !== undefined ? [`${prefix}error: ${run.error}`] : []),
+            ...run.scores.flatMap((score) => score.error !== undefined ? [`${prefix}metric ${score.name} failed: ${score.error}`] : []),
+        ];
+    });
+}
 //# sourceMappingURL=format.js.map

package/dist/reporters/format.js.map

@@ -1 +1 @@
-{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,MAE7B;IACC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC"}
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,MAE7B;IACC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,IAAI,CAAC,KAAa;IAChC,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,GAAG,CAAC,CAAS;IAC3B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AASD;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,OAAO;YACL,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9B,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,UAAU,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAC1F;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/reporters/github.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA2C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAED,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA2C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}

package/dist/reporters/github.js

@@ -1,5 +1,5 @@
 import { appendFileSync } from 'node:fs';
-import { caseFlags, caseLabel, trialsPerCase } from './format.js';
+import { caseFlags, caseLabel, cell, num, trialsPerCase } from './format.js';
 /** Escapes the message of a workflow command */
 function escapeData(value) {
     return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
@@ -31,21 +31,6 @@ export function githubAnnotations(result) {
     }
     return lines;
 }
-/**
- * A table cell of the step summary: any line break (a lone `\r` too) becomes `<br>` so the row
- * holds, and `<` and `&` are escaped so no tag or comment in an answer hides the rest
- */
-function cell(value) {
-    return value
-        .replace(/&/g, '&amp;')
-        .replace(/</g, '&lt;')
-        .replace(/\\/g, '\\\\')
-        .replace(/\|/g, '\\|')
-        .replace(/\r\n|\r|\n/g, '<br>');
-}
-function num(n) {
-    return Number.isInteger(n) ? String(n) : n.toFixed(4);
-}
 /** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
 export function githubSummary(result) {
     const { stats } = result;

package/dist/reporters/github.js.map

@@ -1 +1 @@
-{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE1E,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}

package/dist/reporters/html-format.d.ts

@@ -0,0 +1,11 @@
+/** Escapes text for HTML content and attribute values */
+export declare function escapeHtml(value: string): string;
+/** Text that may be long or span lines: as it is when short, folded in `<details>` otherwise */
+export declare function foldable(value: string, max?: number): string;
+/** A self-contained HTML page: inline CSS, no scripts and no external assets */
+export declare function htmlDocument(title: string, body: string): string;
+/** A table, its cells already HTML; `numeric` marks the columns aligned right */
+export declare function htmlTable(headers: readonly string[], rows: readonly (readonly string[])[], numeric?: ReadonlySet<number>): string;
+/** A row of figures, each value already HTML */
+export declare function htmlTiles(tiles: ReadonlyArray<readonly [label: string, value: string]>): string;
+//# sourceMappingURL=html-format.d.ts.map

package/dist/reporters/html-format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-format.d.ts","sourceRoot":"","sources":["../../src/reporters/html-format.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,gGAAgG;AAChG,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAAY,GAAG,MAAM,CAQjE;AAyDD,gFAAgF;AAChF,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAkBhE;AAED,iFAAiF;AACjF,wBAAgB,SAAS,CACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,IAAI,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,EACpC,OAAO,GAAE,WAAW,CAAC,MAAM,CAAa,GACvC,MAAM,CAWR;AAED,gDAAgD;AAChD,wBAAgB,SAAS,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,CAM/F"}