@namzu/sdk 62.13.0 → 62.15.0

package/dist/eval/compare.d.ts

@@ -1,5 +1,7 @@
 import { type DifferenceIntervals } from './bootstrap.js';
 import { type CaseRuns } from './report-stats.js';
+import { type SettingDifference } from './settings.js';
+import { type SplitFilter } from './split.js';
 /**
  * Two eval reports, side by side, paired by suite and case id.
  *
@@ -18,6 +20,19 @@ export interface LoadedSuite {
     readonly repeats: number;
     /** Whatever the file said about how the suite ran; absent in an older file. */
     readonly facts: Readonly<Record<string, unknown>> | undefined;
+    /** The settings snapshot the suite ran under; absent in a report written before there were any. */
+    readonly settings: LoadedSettings | undefined;
+    /** Which split the suite ran; absent in a report written before there were any. */
+    readonly split: SplitFilter | undefined;
+}
+export interface LoadedSettings {
+    readonly name: string;
+    readonly digest: string;
+    readonly values: Readonly<Record<string, unknown>>;
+    /** False when the suite never asked for the settings. Absent is read as unknown. */
+    readonly read: boolean | undefined;
+    /** Paths the suite asked for, `*` for the whole object; undefined when the report does not say. */
+    readonly readKeys: readonly string[] | undefined;
 }
 export interface LoadedReportFile {
     readonly label: string;
@@ -99,9 +114,48 @@ export interface EvalComparison {
         readonly sentence: string;
         /** Why, in plain words. */
         readonly detail: string;
+        /**
+         * Why no winner is named although the numbers might say one. Empty when
+         * the comparison is on the held-out split, between different settings, over
+         * the same tasks: the one case in which it speaks.
+         */
+        readonly withheld: readonly WithheldReason[];
     };
     readonly disclosure: DisclosureComparison;
+    /** The settings each report ran under, and what differs between the two. */
+    readonly settings: SettingsComparison;
+    /** The split each report ran. Null when a report does not say (an older one). */
+    readonly split: {
+        readonly a: SideSplit;
+        readonly b: SideSplit;
+    };
+}
+/** `mixed` when the suites of one report ran different splits. */
+export type SideSplit = SplitFilter | 'mixed' | null;
+export interface SnapshotRef {
+    readonly name: string;
+    readonly digest: string;
+}
+export interface SettingsComparison {
+    /** Null when the report has no snapshot (it was written before reports carried one). */
+    readonly a: SnapshotRef | null;
+    readonly b: SnapshotRef | null;
+    /** Null when either side has no snapshot. */
+    readonly identical: boolean | null;
+    /** Key by key; empty when the snapshots are identical or either is missing. */
+    readonly differences: readonly SettingDifference[];
+    /** Suites that ran under settings but never read them, so the settings cannot have moved them. */
+    readonly notRead: {
+        readonly a: readonly string[];
+        readonly b: readonly string[];
+    };
+    /** Settings that differ between the two snapshots and that no shared suite on either side read. */
+    readonly unreadKeys: readonly string[];
+    /** Whether at least one differing setting was read by a shared suite. Null when there is no difference to read. */
+    readonly anyDifferenceRead: boolean | null;
 }
+/** Why a comparison does not name a winner, besides the numbers not supporting one. */
+export type WithheldReason = 'tune-split' | 'identical-settings' | 'no-snapshot' | 'tasks-differ' | 'not-held-out' | 'settings-not-read';
 export declare function compareEvalReports(a: LoadedReportFile, b: LoadedReportFile, options?: {
     seed?: number;
 }): EvalComparison;

package/dist/eval/compare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AAGnG;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAoCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACvB,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;CACzC;AAaD,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAgFhB;AAyDD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAqC9D"}
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AACnG,OAAO,EAAE,KAAK,iBAAiB,EAAgB,MAAM,eAAe,CAAA;AACpE,OAAO,EAAiB,KAAK,WAAW,EAAE,MAAM,YAAY,CAAA;AAG5D;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;IAC7D,mGAAmG;IACnG,QAAQ,CAAC,QAAQ,EAAE,cAAc,GAAG,SAAS,CAAA;IAC7C,mFAAmF;IACnF,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,SAAS,CAAA;CACvC;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAClD,oFAAoF;IACpF,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,SAAS,CAAA;IAClC,mGAAmG;IACnG,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAA;CAChD;AAuBD,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAsCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB;;;;WAIG;QACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;KAC5C,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;IACzC,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAA;KAAE,CAAA;CAChE;AAED,kEAAkE;AAClE,MAAM,MAAM,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,IAAI,CAAA;AAEpD,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IAClC,wFAAwF;IACxF,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAA;IAC9B,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAA;IAC9B,6CAA6C;IAC7C,QAAQ,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAAA;IAClC,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAClD,kGAAkG;IAClG,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAA;IAClF,mGAAmG;IACnG,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;IACtC,mHAAmH;IACnH,QAAQ,CAAC,iBAAiB,EAAE,OAAO,GAAG,IAAI,CAAA;CAC1C;AAED,uFAAuF;AACvF,MAAM,MAAM,cAAc,GACvB,YAAY,GACZ,oBAAoB,GACpB,aAAa,GACb,cAAc,GACd,cAAc,GACd,mBAAmB,CAAA;AAatB,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAwFhB;AAuLD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAgE9D"}

package/dist/eval/compare.js

@@ -1,6 +1,30 @@
 import { bootstrapPairedDifference, exactSignTestP, pooledMeanAndRate, validateSeed, } from './bootstrap.js';
 import { UNKNOWN } from './disclosure.js';
 import { duplicateRun, groupRuns, passedEveryRun, tallyOf } from './report-stats.js';
+import { diffSettings } from './settings.js';
+import { SPLIT_FILTERS } from './split.js';
+function loadSettings(value) {
+    if (!isRecord(value))
+        return undefined;
+    if (typeof value.name !== 'string' || typeof value.digest !== 'string')
+        return undefined;
+    return {
+        name: value.name,
+        digest: value.digest,
+        values: isRecord(value.values) ? value.values : {},
+        read: typeof value.read === 'boolean' ? value.read : undefined,
+        readKeys: Array.isArray(value.readKeys)
+            ? value.readKeys.filter((k) => typeof k === 'string')
+            : undefined,
+    };
+}
+function loadSplit(value) {
+    if (!isRecord(value))
+        return undefined;
+    return SPLIT_FILTERS.includes(value.requested)
+        ? value.requested
+        : undefined;
+}
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
@@ -39,6 +63,8 @@ export function loadEvalReportFile(json, label) {
             cases: groupRuns(runs),
             repeats: typeof report.repeats === 'number' ? report.repeats : 1,
             facts: isRecord(report.disclosure) ? report.disclosure : undefined,
+            settings: loadSettings(report.settings),
+            split: loadSplit(report.taskSplit),
         });
     }
     return { label, suites };
@@ -211,7 +237,12 @@ export function compareEvalReports(a, b, options = {}) {
         pValue: exactSignTestP(onlyBPasses, onlyAPasses),
         test: 'exact two-sided sign test on the cases that flipped',
     };
-    const verdict = decide(difference.mean, pairs.length, flips);
+    const settings = compareSettings(a, b, sharedSuiteIds(a, b));
+    const split = { a: splitOf(a), b: splitOf(b) };
+    const withheld = withheldReasons(settings, split, onlyInA.length + onlyInB.length);
+    const verdict = withheld.length > 0
+        ? withhold(withheld, settings, onlyInA.length, onlyInB.length, difference.mean, pairs.length)
+        : { ...decide(difference.mean, pairs.length, flips), withheld: [] };
     const sharedSuites = a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
     return {
         seed,
@@ -227,6 +258,99 @@ export function compareEvalReports(a, b, options = {}) {
         flips,
         verdict,
         disclosure: compareDisclosure(a, b, sharedSuites),
+        settings,
+        split,
+    };
+}
+function sharedSuiteIds(a, b) {
+    return a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
+}
+function snapshotOf(file) {
+    return file.suites.find((s) => s.settings !== undefined)?.settings;
+}
+function splitOf(file) {
+    const requested = new Set(file.suites.map((s) => s.split));
+    if (requested.size === 0 || requested.has(undefined))
+        return null;
+    return requested.size === 1 ? [...requested][0] : 'mixed';
+}
+function compareSettings(a, b, shared) {
+    const left = snapshotOf(a);
+    const right = snapshotOf(b);
+    const ref = (x) => x === undefined ? null : { name: x.name, digest: x.digest };
+    const ignored = (file) => file.suites
+        .filter((s) => shared.includes(s.id) &&
+        s.settings?.read === false &&
+        Object.keys(s.settings.values).length > 0)
+        .map((s) => s.id);
+    const identical = left && right ? left.digest === right.digest : null;
+    const differences = left && right && !identical ? diffSettings(left.values, right.values) : [];
+    const readers = [a, b].flatMap((file) => file.suites.filter((s) => shared.includes(s.id) && s.settings !== undefined));
+    const wasRead = (key) => readers.some((s) => {
+        const st = s.settings;
+        if (st.read === false)
+            return false;
+        // A report that does not say which keys it read is taken to have read all.
+        if (st.readKeys === undefined)
+            return true;
+        return st.readKeys.some((k) => k === '*' || k === key || key.startsWith(`${k}.`) || k.startsWith(`${key}.`));
+    });
+    const unreadKeys = differences.map((d) => d.key).filter((k) => !wasRead(k));
+    return {
+        a: ref(left),
+        b: ref(right),
+        identical,
+        differences,
+        notRead: { a: ignored(a), b: ignored(b) },
+        unreadKeys,
+        anyDifferenceRead: differences.length === 0 ? null : unreadKeys.length < differences.length,
+    };
+}
+function withheldReasons(settings, split, taskMismatches) {
+    const out = [];
+    if (settings.a === null || settings.b === null)
+        out.push('no-snapshot');
+    if (taskMismatches > 0)
+        out.push('tasks-differ');
+    // The most informative reason first: identical settings say the whole
+    // difference is noise, whatever split they ran on.
+    if (settings.identical === true)
+        out.push('identical-settings');
+    // Settings no shared suite read cannot have moved a number: whatever the
+    // difference between the runs, it is not the settings.
+    if (settings.identical === false && settings.anyDifferenceRead === false) {
+        out.push('settings-not-read');
+    }
+    const tune = split.a === 'tune' || split.b === 'tune';
+    if (tune)
+        out.push('tune-split');
+    else if (settings.a !== null &&
+        settings.b !== null &&
+        (split.a !== 'held-out' || split.b !== 'held-out')) {
+        out.push('not-held-out');
+    }
+    return out;
+}
+function short(digest) {
+    return digest.slice(0, 12);
+}
+function withhold(reasons, settings, onlyA, onlyB, mean, paired) {
+    const text = {
+        'tune-split': 'these are the tasks the settings were chosen on; run the held-out split to judge',
+        'identical-settings': `the two reports ran under identical settings (${settings.a?.name ?? 'unnamed'}, ${short(settings.a?.digest ?? '')}), so any difference between them is run-to-run noise`,
+        'no-snapshot': 'a report has no settings snapshot (it was written before reports named their settings), so what the two runs changed is not known',
+        'tasks-differ': `the two reports ran different tasks (${onlyA} only in A, ${onlyB} only in B), so their numbers are not about the same tasks`,
+        'settings-not-read': 'no suite the two reports share read any of the settings that differ between them, so the settings cannot have changed these numbers',
+        'not-held-out': 'a report ran on all tasks or a mix of splits, which include the tasks the settings were chosen on; run the held-out split to judge',
+    };
+    const numbers = mean.computed
+        ? `the paired difference in mean score over ${paired} paired cases is ${signed(mean.estimate)}, 95% interval [${mean.low.toFixed(3)}, ${mean.high.toFixed(3)}]`
+        : `no interval: ${mean.reason}`;
+    return {
+        claim: 'none',
+        sentence: `no winner is named: ${text[reasons[0]]}`,
+        detail: `${[...reasons.slice(1).map((r) => text[r]), numbers].join('; ')}.`,
+        withheld: reasons,
     };
 }
 /** The sign test must be at least this convincing before the sentence names a winner. */
@@ -302,6 +426,27 @@ export function formatEvalComparison(c) {
         `  (${c.verdict.detail})`,
         '',
     ];
+    const ref = (x) => x === null ? 'none recorded' : `"${x.name}" (sha256 ${short(x.digest)})`;
+    lines.push(`Settings: A ${ref(c.settings.a)}, B ${ref(c.settings.b)}. Split: A ${c.split.a ?? 'not recorded'}, B ${c.split.b ?? 'not recorded'}.`);
+    if (c.settings.identical === true) {
+        lines.push('The two runs ran under identical settings.');
+    }
+    else if (c.settings.differences.length > 0) {
+        lines.push('What differs in the settings:');
+        const show = (v, has) => (has ? JSON.stringify(v) : 'not stated');
+        for (const x of c.settings.differences) {
+            lines.push(`  ${x.key}: A ${show(x.a, 'a' in x)}, B ${show(x.b, 'b' in x)}`);
+        }
+    }
+    for (const side of ['a', 'b']) {
+        if (c.settings.notRead[side].length > 0) {
+            lines.push(`Suites that ran under settings but did not read them (${side.toUpperCase()}): ${c.settings.notRead[side].join(', ')}`);
+        }
+    }
+    if (c.settings.unreadKeys.length > 0) {
+        lines.push(`Settings that differ and that no shared suite read: ${c.settings.unreadKeys.join(', ')}`);
+    }
+    lines.push('');
     const d = c.disclosure;
     if (d.differences.length === 0) {
         lines.push('What the two reports say about how they were produced: no stated difference.');

package/dist/eval/compare.js.map

@@ -1 +1 @@
-{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AA6BnG,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SAClE,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AAmDD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC5D,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;KACjD,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CAAC,IAAc,EAAE,MAAc,EAAE,KAAY;IAC3D,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AACnG,OAAO,EAA0B,YAAY,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,EAAE,aAAa,EAAoB,MAAM,YAAY,CAAA;AAsC5D,SAAS,YAAY,CAAC,KAAc;IACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IACtC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IACxF,OAAO;QACN,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAClD,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC9D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YACtC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YAClE,CAAC,CAAC,SAAS;KACZ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IACtC,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAwB,CAAC;QAC5D,CAAC,CAAE,KAAK,CAAC,SAAyB;QAClC,CAAC,CAAC,SAAS,CAAA;AACb,CAAC;AAOD,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAClE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC;YACvC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SAClC,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AA8FD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAClF,MAAM,OAAO,GACZ,QAAQ,CAAC,MAAM,GAAG,CAAC;QAClB,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;QAC7F,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,QAAQ,EAAE,EAAsB,EAAE,CAAA;IACzF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;QACjD,QAAQ;QACR,KAAK;KACL,CAAA;AACF,CAAC;AAED,SAAS,cAAc,CAAC,CAAmB,EAAE,CAAmB;IAC/D,OAAO,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;AACnF,CAAC;AAED,SAAS,UAAU,CAAC,IAAsB;IACzC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,QAAQ,CAAA;AACnE,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB;IACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAA;IACjE,OAAO,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAiB,CAAC,CAAC,CAAC,OAAO,CAAA;AAC3E,CAAC;AAED,SAAS,eAAe,CACvB,CAAmB,EACnB,CAAmB,EACnB,MAAyB;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;IAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,GAAG,GAAG,CAAC,CAA6B,EAAsB,EAAE,CACjE,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IAC5D,MAAM,OAAO,GAAG,CAAC,IAAsB,EAAE,EAAE,CAC1C,IAAI,CAAC,MAAM;SACT,MAAM,CACN,CAAC,CAAC,EAAE,EAAE,CACL,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,CAAC,CAAC,QAAQ,EAAE,IAAI,KAAK,KAAK;QAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1C;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACnB,MAAM,SAAS,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACrE,MAAM,WAAW,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9F,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAC5E,CAAA;IACD,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE,CACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QAClB,MAAM,EAAE,GAAG,CAAC,CAAC,QAA0B,CAAA;QACvC,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO,KAAK,CAAA;QACnC,2EAA2E;QAC3E,IAAI,EAAE,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAA;QAC1C,OAAO,EAAE,CAAC,QAAQ,CAAC,IAAI,CACtB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CACnF,CAAA;IACF,CAAC,CAAC,CAAA;IACH,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3E,OAAO;QACN,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;QACZ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;QACb,SAAS;QACT,WAAW;QACX,OAAO,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE;QACzC,UAAU;QACV,iBAAiB,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;KAC3F,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CACvB,QAA4B,EAC5B,KAAqC,EACrC,cAAsB;IAEtB,MAAM,GAAG,GAAqB,EAAE,CAAA;IAChC,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACvE,IAAI,cAAc,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAChD,sEAAsE;IACtE,mDAAmD;IACnD,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IAC/D,yEAAyE;IACzE,uDAAuD;IACvD,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,IAAI,QAAQ,CAAC,iBAAiB,KAAK,KAAK,EAAE,CAAC;QAC1E,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;IAC9B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,KAAK,MAAM,IAAI,KAAK,CAAC,CAAC,KAAK,MAAM,CAAA;IACrD,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;SAC3B,IACJ,QAAQ,CAAC,CAAC,KAAK,IAAI;QACnB,QAAQ,CAAC,CAAC,KAAK,IAAI;QACnB,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,EACjD,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,KAAK,CAAC,MAAc;IAC5B,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,QAAQ,CAChB,OAAkC,EAClC,QAA4B,EAC5B,KAAa,EACb,KAAa,EACb,IAAc,EACd,MAAc;IAEd,MAAM,IAAI,GAAmC;QAC5C,YAAY,EACX,kFAAkF;QACnF,oBAAoB,EAAE,iDAAiD,QAAQ,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,uDAAuD;QAC/L,aAAa,EACZ,mIAAmI;QACpI,cAAc,EAAE,wCAAwC,KAAK,eAAe,KAAK,4DAA4D;QAC7I,mBAAmB,EAClB,qIAAqI;QACtI,cAAc,EACb,oIAAoI;KACrI,CAAA;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ;QAC5B,CAAC,CAAC,4CAA4C,MAAM,oBAAoB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QAC/J,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAA;IAChC,OAAO;QACN,KAAK,EAAE,MAAM;QACb,QAAQ,EAAE,uBAAuB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAmB,CAAC,EAAE;QACrE,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC3E,QAAQ,EAAE,OAAO;KACjB,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CACd,IAAc,EACd,MAAc,EACd,KAAY;IAEZ,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,GAAG,GAAG,CAAC,CAAqB,EAAE,EAAE,CACrC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAA;IACzE,KAAK,CAAC,IAAI,CACT,eAAe,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,cAAc,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,cAAc,GAAG,CACtI,CAAA;IACD,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAA;IACzD,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,GAAY,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAA;QACnF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAA;QAC7E,CAAC;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAU,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CACT,yDAAyD,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtH,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CACT,uDAAuD,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACd,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/experiment.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAUtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAsE3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA4F7D"}
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAoBtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAgG3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA8G7D"}

package/dist/eval/experiment.js

@@ -1,7 +1,9 @@
 import { EvalUsageError, validateSeed } from './bootstrap.js';
 import { buildDisclosure, describeDisclosure } from './disclosure.js';
-import { describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
+import { clarifyDeclared, describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
 import { currentEvalRunOptions } from './run-options.js';
+import { defaultSettingsSnapshot } from './settings.js';
+import { HELD_OUT_FRACTION, TASK_SPLIT_SEED, assignSplit, taskId, validateSplitFilter, } from './split.js';
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 const MAX_REPEATS = 100;
 const MAX_CASE_TIMEOUT_MS = 2_147_483_647;
@@ -30,8 +32,27 @@ export async function runExperiment(config) {
     const alike = duplicateName(config.cases.map((c) => c.name));
     if (alike !== undefined)
         throw new Error(duplicateMessage(config.name, alike));
+    // Which tasks run: every one, or the half assigned to one split. The
+    // assignment is a function of the task's identity, so it does not move.
+    const requested = validateSplitFilter(options.split);
+    const assigned = config.cases.map((c) => ({
+        evalCase: c,
+        ...assignSplit(config.name, c.name, c.split),
+    }));
+    const chosen = assigned.filter((t) => requested === 'all' || t.split === requested);
+    const cases = chosen.map((t) => t.evalCase);
+    const taskSplit = {
+        requested,
+        seed: TASK_SPLIT_SEED,
+        heldOutFraction: HELD_OUT_FRACTION,
+        tasks: chosen.map((t) => ({
+            id: taskId(config.name, t.evalCase.name),
+            split: t.split,
+            declared: t.declared,
+        })),
+    };
     // Case-major, so a case's repeats sit together in the report.
-    const total = config.cases.length * repeats;
+    const total = cases.length * repeats;
     const results = new Array(total);
     let cursor = 0;
     const worker = async () => {
@@ -39,7 +60,7 @@ export async function runExperiment(config) {
             const index = cursor++;
             if (index >= total)
                 return;
-            const evalCase = config.cases[Math.floor(index / repeats)];
+            const evalCase = cases[Math.floor(index / repeats)];
             const result = await evaluateCase(config, evalCase, threshold, timeoutMs);
             result.repeat = (index % repeats) + 1;
             if (evalCase.split)
@@ -77,7 +98,13 @@ export async function runExperiment(config) {
         intervals: stats.intervals,
         ...(repeats > 1 ? { caseSummaries: stats.caseSummaries, reliability: stats.reliability } : {}),
         splits: stats.splits,
-        disclosure: buildDisclosure(config.disclosure, { cases: config.cases.length, repeats, seed }, options, new Date().toISOString()),
+        disclosure: buildDisclosure(config.disclosure, { cases: cases.length, repeats, seed }, options, new Date().toISOString()),
+        settings: {
+            ...(options.settings ?? defaultSettingsSnapshot()),
+            read: options.settingsUse?.read ?? false,
+            readKeys: [...(options.settingsUse?.keys ?? [])].sort(),
+        },
+        taskSplit,
     };
 }
 function validateRepeats(value) {
@@ -340,11 +367,24 @@ export function formatReport(report) {
         }
     }
     if (report.splits) {
-        for (const line of describeSplits(report.splits))
-            lines.push(`  ${line}`);
+        for (const line of describeSplits(report.splits)) {
+            // The task split is the held-out one: the line about cases a suite
+            // declared as developed against would contradict the run's own header.
+            if (report.taskSplit?.requested === 'held-out' && line.startsWith('no held-out cases'))
+                continue;
+            lines.push(`  ${report.taskSplit ? clarifyDeclared(line) : line}`);
+        }
     }
     if (report.disclosure)
         lines.push(`  ${describeDisclosure(report.disclosure)}`);
+    if (report.settings) {
+        const r = report.settings;
+        lines.push(`  settings: "${r.name}", sha256 ${r.digest.slice(0, 12)}${r.read || Object.keys(r.values).length === 0 ? '' : ' (this suite does not read them)'}`);
+    }
+    if (report.taskSplit) {
+        const t = report.taskSplit;
+        lines.push(`  split: ${t.requested}, ${t.tasks.length} task${t.tasks.length === 1 ? '' : 's'} ran`);
+    }
     lines.push('');
     // Surfaced above the failures, because an inconclusive case means the
     // harness is broken and every number below it is measured over less

package/dist/eval/experiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AAWxD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,8DAA8D;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IAC3C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAE9E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAC7C,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACvD,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAYnB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,qEAAqE;IACrE,wEAAwE;IACxE,MAAM,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,QAAQ,EAAE,CAAC;QACX,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;KAC5C,CAAC,CAAC,CAAA;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAA;IACnF,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAoB;QAClC,SAAS;QACT,IAAI,EAAE,eAAe;QACrB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACpB,CAAC,CAAC;KACH,CAAA;IAED,8DAA8D;IAC9D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IACpC,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAEvE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EACtC,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;QACD,QAAQ,EAAE;YACT,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,uBAAuB,EAAE,CAAC;YAClD,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,IAAI,KAAK;YACxC,QAAQ,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;SACvD;QACD,SAAS;KACT,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,mEAAmE;YACnE,uEAAuE;YACvE,IAAI,MAAM,CAAC,SAAS,EAAE,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;gBACrF,SAAQ;YACT,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAA;QACzB,KAAK,CAAC,IAAI,CACT,gBAAgB,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kCAAkC,EAAE,CACnJ,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAA;QAC1B,KAAK,CAAC,IAAI,CACT,YAAY,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CACvF,CAAA;IACF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/index.d.ts

@@ -11,13 +11,17 @@ export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL,
 export type { BootstrapMethod, CaseIntervals, CaseTally, DifferenceIntervals, Interval, PairedTally, } from './bootstrap.js';
 export type { ReportDisclosure, RoleDisclosure, RoutingDisclosure, RunMode, SuiteFacts, } from './disclosure.js';
 export { sdkVersion } from './disclosure.js';
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { currentEvalRunOptions, currentEvalSettings, evalSetting, withEvalRunOptions, } from './run-options.js';
+export { DEFAULT_SETTINGS_NAME, SETTINGS_DIGEST_ALGORITHM, buildSettingsSnapshot, canonicalJson, defaultSettingsSnapshot, diffSettings, findSecretIn, validateSettingsName, } from './settings.js';
+export type { ReportSettings, SettingDifference, SettingsSnapshot } from './settings.js';
+export { HELD_OUT_FRACTION, SPLIT_FILTERS, TASK_SPLIT_SEED, assignSplit, hashedSplit, taskId, validateSplitFilter, } from './split.js';
+export type { ReportTaskSplit, SplitFilter, TaskSplit } from './split.js';
 export type { EvalRunOptions } from './run-options.js';
 export { formatRunSummary, summarizeRun } from './run-summary.js';
 export type { RunDisclosure, RunSummary } from './run-summary.js';
 export type { RepeatReliability, SplitSummary } from './report-stats.js';
 export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
-export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSuite, SideNumbers, } from './compare.js';
+export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSettings, LoadedSuite, SettingsComparison, SideSplit, SnapshotRef, WithheldReason, SideNumbers, } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 export type { ScoreUncertainty } from './uncertainty.js';
 export type { CaseRepeatSummary, CaseResult, EvalSplit, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EACN,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,YAAY,EACZ,YAAY,EACZ,oBAAoB,GACpB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACxF,OAAO,EACN,iBAAiB,EACjB,aAAa,EACb,eAAe,EACf,WAAW,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACzE,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,cAAc,EACd,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}

package/dist/eval/index.js

@@ -5,7 +5,9 @@ export { judgeScorer } from './judge.js';
 export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
 export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
 export { sdkVersion } from './disclosure.js';
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { currentEvalRunOptions, currentEvalSettings, evalSetting, withEvalRunOptions, } from './run-options.js';
+export { DEFAULT_SETTINGS_NAME, SETTINGS_DIGEST_ALGORITHM, buildSettingsSnapshot, canonicalJson, defaultSettingsSnapshot, diffSettings, findSecretIn, validateSettingsName, } from './settings.js';
+export { HELD_OUT_FRACTION, SPLIT_FILTERS, TASK_SPLIT_SEED, assignSplit, hashedSplit, taskId, validateSplitFilter, } from './split.js';
 export { formatRunSummary, summarizeRun } from './run-summary.js';
 export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAa3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EACN,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,YAAY,EACZ,YAAY,EACZ,oBAAoB,GACpB,MAAM,eAAe,CAAA;AAEtB,OAAO,EACN,iBAAiB,EACjB,aAAa,EACb,eAAe,EACf,WAAW,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAkB3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/eval/report-stats.d.ts

@@ -74,4 +74,10 @@ export declare function describeIntervals(args: {
 }): string[];
 /** The line that says which splits a suite has, and says so plainly when none is held out. */
 export declare function describeSplits(splits: RunStats['splits']): string[];
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export declare function clarifyDeclared(line: string): string;
 //# sourceMappingURL=report-stats.d.ts.map