@namzu/sdk 62.14.0 → 62.15.0

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 62.15.0
+
+### Minor Changes
+
+- 67dd0d0: A comparison of two eval reports no longer names a winner for settings that no suite read. `ReportSettings` gains `readKeys` (the settings a suite asked for through `evalSetting` or `currentEvalSettings`, `*` for the whole object), `EvalComparison.settings` gains `unreadKeys` and `anyDifferenceRead`, and `WithheldReason` gains `settings-not-read`, given when the snapshots differ and no shared suite read a key that differs. A script that matched on the exhaustive list of reasons needs one more case; a comparison that used to name a winner between settings the suites ignored now says `no winner is named`. A password written into a URL is now refused in a settings snapshot, as a key is. When a report records a task split, the suite's own "no held-out cases" line says it is the suite's declaration, not the hashed split.
+- 67dd0d0: Eval reports now say which settings they ran under and which split of the tasks they ran, and `compareEvalReports` no longer names a winner it cannot support. Every `ExperimentReport` gains `settings` (a name, a SHA-256 digest of the canonical values, the values, and whether the suite read them) and `taskSplit` (the split asked for, the seed and share, and the id and split of each task that ran); `RunSummary.disclosure` gains `settings` and `split`. Each task is in `tune` or `held-out` by a seeded hash of its suite and case name, or by the `split` the case states (`development` is `tune`); `EvalRunOptions` takes `split` and `settings`, and a suite reads the settings with the new `evalSetting` and `currentEvalSettings`. A settings snapshot refuses any key that names a credential and any value shaped like one.
+
+  What to check: `EvalComparison.verdict` gains `withheld`, and `verdict.claim` is now `none` with a sentence starting `no winner is named` whenever either report was run on the tune split or on all tasks, has no settings snapshot (every report written before this version), ran the same settings as the other, or ran different tasks. A script that read `claim === 'better'` from a comparison of two older reports now gets `none`; to get a claim, run both with `namzu eval --split held-out` under different settings. Every number in a report is unchanged. `EvalComparison` also gains `settings` and `split`.
+
+- b9618bd: Peer messaging can carry the operator's own words from another terminal. New: the `person` sender kind (`PeerSessionKind`), `formatPersonMessage` (a frame that says the text arrived from another terminal of this account and is meant to be the person's, that the kernel did not check who sent it, that it cannot approve anything, change a mode or run a command, and that what the model does because of it passes the same permission checks, with the text quoted under a per-render nonce), the `person-message` runtime-context kind, an optional `conversation` on `deliver` so a host that moved to another conversation can refuse, and a `delivered` value for a delivery notice's `outcome` (`NoticeOutcomeSchema`). `formatPeerMessage` renders a `person` sender through `formatPersonMessage`. A build that predates `person-message` reads such a recorded message as an ordinary user message whose text still carries the kernel's paragraph; a host with its own exhaustive switch over `RuntimeContextMessageKind` or `PeerSessionKind` needs a case for the new members.
+
+  Every process the kernel starts on a session's behalf now has `NAMZU_STARTED_BY_SESSION=1` in its environment: the shell tool and its background jobs, shell hooks, MCP stdio servers (set after any configured value), local-sandbox commands, `LocalExecutionContext` commands and sandbox terminals. New exports: `SESSION_CHILD_ENV`, `withSessionChildMarker(env)` and `startedBySession(env)`. A program that must not be run by a session reads it; a host that builds its own child environment for a process an agent can drive should pass it through `withSessionChildMarker`. Nothing you ran before reads the variable, so nothing breaks unless a tool you ran under a session already used that name.
+
 ## 62.14.0
 
 ### Minor Changes

package/dist/connector/mcp/stdio.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stdio.d.ts","sourceRoot":"","sources":["../../../src/connector/mcp/stdio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACX,iBAAiB,EACjB,uBAAuB,EACvB,YAAY,EACZ,uBAAuB,EACvB,MAAM,gCAAgC,CAAA;AAEvC,OAAO,EAAE,KAAK,MAAM,EAAiB,MAAM,uBAAuB,CAAA;AAyDlE;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAC5B,MAAM,EAAE,IAAI,CAAC,uBAAuB,EAAE,KAAK,GAAG,YAAY,CAAC,EAC3D,MAAM,GAAE,MAAM,CAAC,UAAwB,GACrC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAWxB;AAED,qBAAa,cAAe,YAAW,YAAY;IAgBjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAfxB,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,eAAe,CAAkD;IACzE,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,aAAa,CAAoC;IACzD,OAAO,CAAC,SAAS,CAAQ;IACzB,8DAA8D;IAC9D,OAAO,CAAC,OAAO,CAAQ;IACvB,wEAAwE;IACxE,OAAO,CAAC,UAAU,CAAI;IACtB,6DAA6D;IAC7D,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,GAAG,CAAQ;gBAGD,MAAM,EAAE,uBAAuB,EAChD,GAAG,CAAC,EAAE,MAAM;IAKP,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmDxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuD5B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IAMrB;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAgBR,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAaxF,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,GAAG,IAAI;IAI9D,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIlC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI9C,WAAW,IAAI,OAAO;IAItB,OAAO,CAAC,aAAa;CAiBrB"}
+{"version":3,"file":"stdio.d.ts","sourceRoot":"","sources":["../../../src/connector/mcp/stdio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACX,iBAAiB,EACjB,uBAAuB,EACvB,YAAY,EACZ,uBAAuB,EACvB,MAAM,gCAAgC,CAAA;AAEvC,OAAO,EAAE,KAAK,MAAM,EAAiB,MAAM,uBAAuB,CAAA;AAyDlE;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAC5B,MAAM,EAAE,IAAI,CAAC,uBAAuB,EAAE,KAAK,GAAG,YAAY,CAAC,EAC3D,MAAM,GAAE,MAAM,CAAC,UAAwB,GACrC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAaxB;AAED,qBAAa,cAAe,YAAW,YAAY;IAgBjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAfxB,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,eAAe,CAAkD;IACzE,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,aAAa,CAAoC;IACzD,OAAO,CAAC,SAAS,CAAQ;IACzB,8DAA8D;IAC9D,OAAO,CAAC,OAAO,CAAQ;IACvB,wEAAwE;IACxE,OAAO,CAAC,UAAU,CAAI;IACtB,6DAA6D;IAC7D,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,GAAG,CAAQ;gBAGD,MAAM,EAAE,uBAAuB,EAChD,GAAG,CAAC,EAAE,MAAM;IAKP,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmDxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuD5B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IAMrB;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAgBR,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAaxF,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,GAAG,IAAI;IAI9D,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIlC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI9C,WAAW,IAAI,OAAO;IAItB,OAAO,CAAC,aAAa;CAiBrB"}

package/dist/connector/mcp/stdio.js

@@ -1,4 +1,5 @@
 import { spawn } from 'node:child_process';
+import { SESSION_CHILD_ENV } from '../../process/session-child.js';
 import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js';
 import { resolveLogger } from '../../utils/logger.js';
 import { WINDOWS_CORE_ENV_KEYS, applyEnvironmentOverrides, pickEnvironmentEntries, readEnvironmentEntry, setEnvironmentEntry, } from '../../utils/process-environment.js';
@@ -69,6 +70,8 @@ export function buildChildEnv(config, source = process.env) {
             setEnvironmentEntry(env, found[0], found[1]);
     }
     applyEnvironmentOverrides(env, config.env);
+    // Last, so a configured value cannot take it back: a tool server is started by the session.
+    setEnvironmentEntry(env, SESSION_CHILD_ENV, '1');
     return env;
 }
 export class StdioTransport {

package/dist/connector/mcp/stdio.js.map

@@ -1 +1 @@
-{"version":3,"file":"stdio.js","sourceRoot":"","sources":["../../../src/connector/mcp/stdio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAO7D,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAe,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAClE,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,GACnB,MAAM,oCAAoC,CAAA;AAE3C;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAEhC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,aAAa,GAAsB;IACxC,cAAc;IACd,MAAM;IACN,MAAM;IACN,QAAQ;IACR,UAAU;IACV,IAAI;IACJ,SAAS;IACT,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,SAAS;IACT,0EAA0E;IAC1E,sEAAsE;IACtE,GAAG,qBAAqB;CACxB,CAAA;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAC5B,MAA2D,EAC3D,SAA4B,OAAO,CAAC,GAAG;IAEvC,MAAM,GAAG,GAAG,sBAAsB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACzD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAChD,uEAAuE;QACvE,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,KAAK;YAAE,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,yBAAyB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;IAC1C,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,OAAO,cAAc;IAgBR;IAfV,OAAO,GAAwB,IAAI,CAAA;IACnC,eAAe,GAAgD,EAAE,CAAA;IACjE,aAAa,GAAsB,EAAE,CAAA;IACrC,aAAa,GAAkC,EAAE,CAAA;IACjD,SAAS,GAAG,KAAK,CAAA;IACzB,8DAA8D;IACtD,OAAO,GAAG,KAAK,CAAA;IACvB,wEAAwE;IAChE,UAAU,GAAG,CAAC,CAAA;IACtB,6DAA6D;IACrD,WAAW,GAAG,KAAK,CAAA;IACnB,MAAM,GAAG,EAAE,CAAA;IACX,GAAG,CAAQ;IAEnB,YACkB,MAA+B,EAChD,GAAY;QADK,WAAM,GAAN,MAAM,CAAyB;QAGhD,IAAI,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAA;IAClF,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,sEAAsE;QACtE,yEAAyE;QACzE,mEAAmE;QACnE,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;QACpC,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAA;QACpC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QACpB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAEhB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE;YAChE,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC;YAC/B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAC/B,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QAEpB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC1C,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAM;YAC1D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;YACtC,IAAI,CAAC,aAAa,EAAE,CAAA;QACrB,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QACjE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QACnE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAE3E,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAClC,kBAAkB,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;aAClD,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QAClE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAE1E,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;gBAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YAC/C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;YACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAA;YAC3E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;QAEF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAEjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE;YACzC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YACxC,gBAAgB,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;SACpD,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,oEAAoE;YACpE,8DAA8D;YAC9D,oEAAoE;YACpE,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,aAAa,EAAE,CAAA;YAC3C,OAAM;QACP,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAA;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAEhB,mEAAmE;QACnE,mEAAmE;QACnE,uEAAuE;QACvE,uEAAuE;QACvE,+DAA+D;QAC/D,6BAA6B;QAC7B,EAAE;QACF,qEAAqE;QACrE,kEAAkE;QAClE,sEAAsE;QACtE,uDAAuD;QACvD,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;YACxB,OAAM;QACP,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,OAAO,GAAG,KAAK,CAAA;YACnB,MAAM,MAAM,GAAG,GAAS,EAAE;gBACzB,IAAI,OAAO;oBAAE,OAAM;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,YAAY,CAAC,QAAQ,CAAC,CAAA;gBACtB,YAAY,CAAC,MAAM,CAAC,CAAA;gBACpB,OAAO,EAAE,CAAA;YACV,CAAC,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;gBAC1D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAA;YAC5E,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAA;YACzD,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAA;YAClB,MAAM,CAAC,KAAK,EAAE,EAAE,CAAA;QACjB,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,qEAAqE;QACrE,uEAAuE;QACvE,yEAAyE;IAC1E,CAAC;IAED;;;;;;;;OAQG;IACK,aAAa;QACpB,IAAI,CAAC,eAAe,GAAG,EAAE,CAAA;QACzB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA;QACvB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA;IACxB,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,UAAkB,EAAE,IAAuB,EAAE,KAAa;QACxE,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO;YAAE,OAAM;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QACtB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAA;QACxC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAA;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;YAC/D,KAAK,MAAM,OAAO,IAAI,aAAa;gBAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACpD,OAAM;QACP,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,aAAa;YAAE,OAAO,EAAE,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAA0B,EAAE,OAAiC;QACvE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACjC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;QACvE,CAAC;QACD,wEAAwE;QACxE,6DAA6D;QAC7D,uEAAuE;QACvE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACjC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;QAC3C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAED,SAAS,CAAC,OAA6C;QACtD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACnC,CAAC;IAED,OAAO,CAAC,OAAmB;QAC1B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IAED,OAAO,CAAC,OAA+B;QACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IAED,WAAW;QACV,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEO,aAAa;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACrC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAC3B,IAAI,CAAC,OAAO;gBAAE,SAAQ;YACtB,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAsB,CAAA;gBACxD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,eAAe;oBAAE,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACR,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0CAA0C,EAAE;oBACzD,wBAAwB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBAC/C,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;CACD"}
+{"version":3,"file":"stdio.js","sourceRoot":"","sources":["../../../src/connector/mcp/stdio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AAOlE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAe,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAClE,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,GACnB,MAAM,oCAAoC,CAAA;AAE3C;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAEhC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,aAAa,GAAsB;IACxC,cAAc;IACd,MAAM;IACN,MAAM;IACN,QAAQ;IACR,UAAU;IACV,IAAI;IACJ,SAAS;IACT,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,SAAS;IACT,0EAA0E;IAC1E,sEAAsE;IACtE,GAAG,qBAAqB;CACxB,CAAA;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAC5B,MAA2D,EAC3D,SAA4B,OAAO,CAAC,GAAG;IAEvC,MAAM,GAAG,GAAG,sBAAsB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACzD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAChD,uEAAuE;QACvE,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,KAAK;YAAE,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,yBAAyB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;IAC1C,4FAA4F;IAC5F,mBAAmB,CAAC,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAA;IAChD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,OAAO,cAAc;IAgBR;IAfV,OAAO,GAAwB,IAAI,CAAA;IACnC,eAAe,GAAgD,EAAE,CAAA;IACjE,aAAa,GAAsB,EAAE,CAAA;IACrC,aAAa,GAAkC,EAAE,CAAA;IACjD,SAAS,GAAG,KAAK,CAAA;IACzB,8DAA8D;IACtD,OAAO,GAAG,KAAK,CAAA;IACvB,wEAAwE;IAChE,UAAU,GAAG,CAAC,CAAA;IACtB,6DAA6D;IACrD,WAAW,GAAG,KAAK,CAAA;IACnB,MAAM,GAAG,EAAE,CAAA;IACX,GAAG,CAAQ;IAEnB,YACkB,MAA+B,EAChD,GAAY;QADK,WAAM,GAAN,MAAM,CAAyB;QAGhD,IAAI,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAA;IAClF,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,sEAAsE;QACtE,yEAAyE;QACzE,mEAAmE;QACnE,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;QACpC,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAA;QACpC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QACpB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAEhB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE;YAChE,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC;YAC/B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAC/B,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QAEpB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC1C,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAM;YAC1D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;YACtC,IAAI,CAAC,aAAa,EAAE,CAAA;QACrB,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QACjE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QACnE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAE3E,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAClC,kBAAkB,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;aAClD,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QAClE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAE1E,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;gBAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YAC/C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;YACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAA;YAC3E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;QAEF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;QAEjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE;YACzC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YACxC,gBAAgB,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;SACpD,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,oEAAoE;YACpE,8DAA8D;YAC9D,oEAAoE;YACpE,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,aAAa,EAAE,CAAA;YAC3C,OAAM;QACP,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAA;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAEhB,mEAAmE;QACnE,mEAAmE;QACnE,uEAAuE;QACvE,uEAAuE;QACvE,+DAA+D;QAC/D,6BAA6B;QAC7B,EAAE;QACF,qEAAqE;QACrE,kEAAkE;QAClE,sEAAsE;QACtE,uDAAuD;QACvD,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;YACxB,OAAM;QACP,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,OAAO,GAAG,KAAK,CAAA;YACnB,MAAM,MAAM,GAAG,GAAS,EAAE;gBACzB,IAAI,OAAO;oBAAE,OAAM;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,YAAY,CAAC,QAAQ,CAAC,CAAA;gBACtB,YAAY,CAAC,MAAM,CAAC,CAAA;gBACpB,OAAO,EAAE,CAAA;YACV,CAAC,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;gBAC1D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAA;YAC5E,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAA;YACzD,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAA;YAClB,MAAM,CAAC,KAAK,EAAE,EAAE,CAAA;QACjB,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,qEAAqE;QACrE,uEAAuE;QACvE,yEAAyE;IAC1E,CAAC;IAED;;;;;;;;OAQG;IACK,aAAa;QACpB,IAAI,CAAC,eAAe,GAAG,EAAE,CAAA;QACzB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA;QACvB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA;IACxB,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,UAAkB,EAAE,IAAuB,EAAE,KAAa;QACxE,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO;YAAE,OAAM;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QACtB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAA;QACxC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAA;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;YAC/D,KAAK,MAAM,OAAO,IAAI,aAAa;gBAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACpD,OAAM;QACP,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,aAAa;YAAE,OAAO,EAAE,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAA0B,EAAE,OAAiC;QACvE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACjC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;QACvE,CAAC;QACD,wEAAwE;QACxE,6DAA6D;QAC7D,uEAAuE;QACvE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACjC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;QAC3C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAED,SAAS,CAAC,OAA6C;QACtD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACnC,CAAC;IAED,OAAO,CAAC,OAAmB;QAC1B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IAED,OAAO,CAAC,OAA+B;QACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IAED,WAAW;QACV,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEO,aAAa;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACrC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAC3B,IAAI,CAAC,OAAO;gBAAE,SAAQ;YACtB,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAsB,CAAA;gBACxD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,eAAe;oBAAE,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACR,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0CAA0C,EAAE;oBACzD,wBAAwB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBAC/C,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;CACD"}

package/dist/eval/compare.d.ts

@@ -1,5 +1,7 @@
 import { type DifferenceIntervals } from './bootstrap.js';
 import { type CaseRuns } from './report-stats.js';
+import { type SettingDifference } from './settings.js';
+import { type SplitFilter } from './split.js';
 /**
  * Two eval reports, side by side, paired by suite and case id.
  *
@@ -18,6 +20,19 @@ export interface LoadedSuite {
     readonly repeats: number;
     /** Whatever the file said about how the suite ran; absent in an older file. */
     readonly facts: Readonly<Record<string, unknown>> | undefined;
+    /** The settings snapshot the suite ran under; absent in a report written before there were any. */
+    readonly settings: LoadedSettings | undefined;
+    /** Which split the suite ran; absent in a report written before there were any. */
+    readonly split: SplitFilter | undefined;
+}
+export interface LoadedSettings {
+    readonly name: string;
+    readonly digest: string;
+    readonly values: Readonly<Record<string, unknown>>;
+    /** False when the suite never asked for the settings. Absent is read as unknown. */
+    readonly read: boolean | undefined;
+    /** Paths the suite asked for, `*` for the whole object; undefined when the report does not say. */
+    readonly readKeys: readonly string[] | undefined;
 }
 export interface LoadedReportFile {
     readonly label: string;
@@ -99,9 +114,48 @@ export interface EvalComparison {
         readonly sentence: string;
         /** Why, in plain words. */
         readonly detail: string;
+        /**
+         * Why no winner is named although the numbers might say one. Empty when
+         * the comparison is on the held-out split, between different settings, over
+         * the same tasks: the one case in which it speaks.
+         */
+        readonly withheld: readonly WithheldReason[];
     };
     readonly disclosure: DisclosureComparison;
+    /** The settings each report ran under, and what differs between the two. */
+    readonly settings: SettingsComparison;
+    /** The split each report ran. Null when a report does not say (an older one). */
+    readonly split: {
+        readonly a: SideSplit;
+        readonly b: SideSplit;
+    };
+}
+/** `mixed` when the suites of one report ran different splits. */
+export type SideSplit = SplitFilter | 'mixed' | null;
+export interface SnapshotRef {
+    readonly name: string;
+    readonly digest: string;
+}
+export interface SettingsComparison {
+    /** Null when the report has no snapshot (it was written before reports carried one). */
+    readonly a: SnapshotRef | null;
+    readonly b: SnapshotRef | null;
+    /** Null when either side has no snapshot. */
+    readonly identical: boolean | null;
+    /** Key by key; empty when the snapshots are identical or either is missing. */
+    readonly differences: readonly SettingDifference[];
+    /** Suites that ran under settings but never read them, so the settings cannot have moved them. */
+    readonly notRead: {
+        readonly a: readonly string[];
+        readonly b: readonly string[];
+    };
+    /** Settings that differ between the two snapshots and that no shared suite on either side read. */
+    readonly unreadKeys: readonly string[];
+    /** Whether at least one differing setting was read by a shared suite. Null when there is no difference to read. */
+    readonly anyDifferenceRead: boolean | null;
 }
+/** Why a comparison does not name a winner, besides the numbers not supporting one. */
+export type WithheldReason = 'tune-split' | 'identical-settings' | 'no-snapshot' | 'tasks-differ' | 'not-held-out' | 'settings-not-read';
 export declare function compareEvalReports(a: LoadedReportFile, b: LoadedReportFile, options?: {
     seed?: number;
 }): EvalComparison;

package/dist/eval/compare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AAGnG;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAoCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACvB,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;CACzC;AAaD,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAgFhB;AAyDD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAqC9D"}
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AACnG,OAAO,EAAE,KAAK,iBAAiB,EAAgB,MAAM,eAAe,CAAA;AACpE,OAAO,EAAiB,KAAK,WAAW,EAAE,MAAM,YAAY,CAAA;AAG5D;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;IAC7D,mGAAmG;IACnG,QAAQ,CAAC,QAAQ,EAAE,cAAc,GAAG,SAAS,CAAA;IAC7C,mFAAmF;IACnF,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,SAAS,CAAA;CACvC;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAClD,oFAAoF;IACpF,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,SAAS,CAAA;IAClC,mGAAmG;IACnG,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAA;CAChD;AAuBD,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAsCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB;;;;WAIG;QACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;KAC5C,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;IACzC,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAA;KAAE,CAAA;CAChE;AAED,kEAAkE;AAClE,MAAM,MAAM,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,IAAI,CAAA;AAEpD,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IAClC,wFAAwF;IACxF,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAA;IAC9B,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAA;IAC9B,6CAA6C;IAC7C,QAAQ,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAAA;IAClC,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAClD,kGAAkG;IAClG,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAA;IAClF,mGAAmG;IACnG,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;IACtC,mHAAmH;IACnH,QAAQ,CAAC,iBAAiB,EAAE,OAAO,GAAG,IAAI,CAAA;CAC1C;AAED,uFAAuF;AACvF,MAAM,MAAM,cAAc,GACvB,YAAY,GACZ,oBAAoB,GACpB,aAAa,GACb,cAAc,GACd,cAAc,GACd,mBAAmB,CAAA;AAatB,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAwFhB;AAuLD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAgE9D"}

package/dist/eval/compare.js

@@ -1,6 +1,30 @@
 import { bootstrapPairedDifference, exactSignTestP, pooledMeanAndRate, validateSeed, } from './bootstrap.js';
 import { UNKNOWN } from './disclosure.js';
 import { duplicateRun, groupRuns, passedEveryRun, tallyOf } from './report-stats.js';
+import { diffSettings } from './settings.js';
+import { SPLIT_FILTERS } from './split.js';
+function loadSettings(value) {
+    if (!isRecord(value))
+        return undefined;
+    if (typeof value.name !== 'string' || typeof value.digest !== 'string')
+        return undefined;
+    return {
+        name: value.name,
+        digest: value.digest,
+        values: isRecord(value.values) ? value.values : {},
+        read: typeof value.read === 'boolean' ? value.read : undefined,
+        readKeys: Array.isArray(value.readKeys)
+            ? value.readKeys.filter((k) => typeof k === 'string')
+            : undefined,
+    };
+}
+function loadSplit(value) {
+    if (!isRecord(value))
+        return undefined;
+    return SPLIT_FILTERS.includes(value.requested)
+        ? value.requested
+        : undefined;
+}
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
@@ -39,6 +63,8 @@ export function loadEvalReportFile(json, label) {
             cases: groupRuns(runs),
             repeats: typeof report.repeats === 'number' ? report.repeats : 1,
             facts: isRecord(report.disclosure) ? report.disclosure : undefined,
+            settings: loadSettings(report.settings),
+            split: loadSplit(report.taskSplit),
         });
     }
     return { label, suites };
@@ -211,7 +237,12 @@ export function compareEvalReports(a, b, options = {}) {
         pValue: exactSignTestP(onlyBPasses, onlyAPasses),
         test: 'exact two-sided sign test on the cases that flipped',
     };
-    const verdict = decide(difference.mean, pairs.length, flips);
+    const settings = compareSettings(a, b, sharedSuiteIds(a, b));
+    const split = { a: splitOf(a), b: splitOf(b) };
+    const withheld = withheldReasons(settings, split, onlyInA.length + onlyInB.length);
+    const verdict = withheld.length > 0
+        ? withhold(withheld, settings, onlyInA.length, onlyInB.length, difference.mean, pairs.length)
+        : { ...decide(difference.mean, pairs.length, flips), withheld: [] };
     const sharedSuites = a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
     return {
         seed,
@@ -227,6 +258,99 @@ export function compareEvalReports(a, b, options = {}) {
         flips,
         verdict,
         disclosure: compareDisclosure(a, b, sharedSuites),
+        settings,
+        split,
+    };
+}
+function sharedSuiteIds(a, b) {
+    return a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
+}
+function snapshotOf(file) {
+    return file.suites.find((s) => s.settings !== undefined)?.settings;
+}
+function splitOf(file) {
+    const requested = new Set(file.suites.map((s) => s.split));
+    if (requested.size === 0 || requested.has(undefined))
+        return null;
+    return requested.size === 1 ? [...requested][0] : 'mixed';
+}
+function compareSettings(a, b, shared) {
+    const left = snapshotOf(a);
+    const right = snapshotOf(b);
+    const ref = (x) => x === undefined ? null : { name: x.name, digest: x.digest };
+    const ignored = (file) => file.suites
+        .filter((s) => shared.includes(s.id) &&
+        s.settings?.read === false &&
+        Object.keys(s.settings.values).length > 0)
+        .map((s) => s.id);
+    const identical = left && right ? left.digest === right.digest : null;
+    const differences = left && right && !identical ? diffSettings(left.values, right.values) : [];
+    const readers = [a, b].flatMap((file) => file.suites.filter((s) => shared.includes(s.id) && s.settings !== undefined));
+    const wasRead = (key) => readers.some((s) => {
+        const st = s.settings;
+        if (st.read === false)
+            return false;
+        // A report that does not say which keys it read is taken to have read all.
+        if (st.readKeys === undefined)
+            return true;
+        return st.readKeys.some((k) => k === '*' || k === key || key.startsWith(`${k}.`) || k.startsWith(`${key}.`));
+    });
+    const unreadKeys = differences.map((d) => d.key).filter((k) => !wasRead(k));
+    return {
+        a: ref(left),
+        b: ref(right),
+        identical,
+        differences,
+        notRead: { a: ignored(a), b: ignored(b) },
+        unreadKeys,
+        anyDifferenceRead: differences.length === 0 ? null : unreadKeys.length < differences.length,
+    };
+}
+function withheldReasons(settings, split, taskMismatches) {
+    const out = [];
+    if (settings.a === null || settings.b === null)
+        out.push('no-snapshot');
+    if (taskMismatches > 0)
+        out.push('tasks-differ');
+    // The most informative reason first: identical settings say the whole
+    // difference is noise, whatever split they ran on.
+    if (settings.identical === true)
+        out.push('identical-settings');
+    // Settings no shared suite read cannot have moved a number: whatever the
+    // difference between the runs, it is not the settings.
+    if (settings.identical === false && settings.anyDifferenceRead === false) {
+        out.push('settings-not-read');
+    }
+    const tune = split.a === 'tune' || split.b === 'tune';
+    if (tune)
+        out.push('tune-split');
+    else if (settings.a !== null &&
+        settings.b !== null &&
+        (split.a !== 'held-out' || split.b !== 'held-out')) {
+        out.push('not-held-out');
+    }
+    return out;
+}
+function short(digest) {
+    return digest.slice(0, 12);
+}
+function withhold(reasons, settings, onlyA, onlyB, mean, paired) {
+    const text = {
+        'tune-split': 'these are the tasks the settings were chosen on; run the held-out split to judge',
+        'identical-settings': `the two reports ran under identical settings (${settings.a?.name ?? 'unnamed'}, ${short(settings.a?.digest ?? '')}), so any difference between them is run-to-run noise`,
+        'no-snapshot': 'a report has no settings snapshot (it was written before reports named their settings), so what the two runs changed is not known',
+        'tasks-differ': `the two reports ran different tasks (${onlyA} only in A, ${onlyB} only in B), so their numbers are not about the same tasks`,
+        'settings-not-read': 'no suite the two reports share read any of the settings that differ between them, so the settings cannot have changed these numbers',
+        'not-held-out': 'a report ran on all tasks or a mix of splits, which include the tasks the settings were chosen on; run the held-out split to judge',
+    };
+    const numbers = mean.computed
+        ? `the paired difference in mean score over ${paired} paired cases is ${signed(mean.estimate)}, 95% interval [${mean.low.toFixed(3)}, ${mean.high.toFixed(3)}]`
+        : `no interval: ${mean.reason}`;
+    return {
+        claim: 'none',
+        sentence: `no winner is named: ${text[reasons[0]]}`,
+        detail: `${[...reasons.slice(1).map((r) => text[r]), numbers].join('; ')}.`,
+        withheld: reasons,
     };
 }
 /** The sign test must be at least this convincing before the sentence names a winner. */
@@ -302,6 +426,27 @@ export function formatEvalComparison(c) {
         `  (${c.verdict.detail})`,
         '',
     ];
+    const ref = (x) => x === null ? 'none recorded' : `"${x.name}" (sha256 ${short(x.digest)})`;
+    lines.push(`Settings: A ${ref(c.settings.a)}, B ${ref(c.settings.b)}. Split: A ${c.split.a ?? 'not recorded'}, B ${c.split.b ?? 'not recorded'}.`);
+    if (c.settings.identical === true) {
+        lines.push('The two runs ran under identical settings.');
+    }
+    else if (c.settings.differences.length > 0) {
+        lines.push('What differs in the settings:');
+        const show = (v, has) => (has ? JSON.stringify(v) : 'not stated');
+        for (const x of c.settings.differences) {
+            lines.push(`  ${x.key}: A ${show(x.a, 'a' in x)}, B ${show(x.b, 'b' in x)}`);
+        }
+    }
+    for (const side of ['a', 'b']) {
+        if (c.settings.notRead[side].length > 0) {
+            lines.push(`Suites that ran under settings but did not read them (${side.toUpperCase()}): ${c.settings.notRead[side].join(', ')}`);
+        }
+    }
+    if (c.settings.unreadKeys.length > 0) {
+        lines.push(`Settings that differ and that no shared suite read: ${c.settings.unreadKeys.join(', ')}`);
+    }
+    lines.push('');
     const d = c.disclosure;
     if (d.differences.length === 0) {
         lines.push('What the two reports say about how they were produced: no stated difference.');

package/dist/eval/compare.js.map

@@ -1 +1 @@
-{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AA6BnG,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SAClE,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AAmDD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC5D,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;KACjD,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CAAC,IAAc,EAAE,MAAc,EAAE,KAAY;IAC3D,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AACnG,OAAO,EAA0B,YAAY,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,EAAE,aAAa,EAAoB,MAAM,YAAY,CAAA;AAsC5D,SAAS,YAAY,CAAC,KAAc;IACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IACtC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IACxF,OAAO;QACN,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAClD,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC9D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YACtC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YAClE,CAAC,CAAC,SAAS;KACZ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IACtC,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAwB,CAAC;QAC5D,CAAC,CAAE,KAAK,CAAC,SAAyB;QAClC,CAAC,CAAC,SAAS,CAAA;AACb,CAAC;AAOD,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAClE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC;YACvC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SAClC,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AA8FD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAClF,MAAM,OAAO,GACZ,QAAQ,CAAC,MAAM,GAAG,CAAC;QAClB,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;QAC7F,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,QAAQ,EAAE,EAAsB,EAAE,CAAA;IACzF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;QACjD,QAAQ;QACR,KAAK;KACL,CAAA;AACF,CAAC;AAED,SAAS,cAAc,CAAC,CAAmB,EAAE,CAAmB;IAC/D,OAAO,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;AACnF,CAAC;AAED,SAAS,UAAU,CAAC,IAAsB;IACzC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,QAAQ,CAAA;AACnE,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB;IACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAA;IACjE,OAAO,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAiB,CAAC,CAAC,CAAC,OAAO,CAAA;AAC3E,CAAC;AAED,SAAS,eAAe,CACvB,CAAmB,EACnB,CAAmB,EACnB,MAAyB;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;IAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,GAAG,GAAG,CAAC,CAA6B,EAAsB,EAAE,CACjE,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IAC5D,MAAM,OAAO,GAAG,CAAC,IAAsB,EAAE,EAAE,CAC1C,IAAI,CAAC,MAAM;SACT,MAAM,CACN,CAAC,CAAC,EAAE,EAAE,CACL,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,CAAC,CAAC,QAAQ,EAAE,IAAI,KAAK,KAAK;QAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1C;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACnB,MAAM,SAAS,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACrE,MAAM,WAAW,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9F,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAC5E,CAAA;IACD,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE,CACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QAClB,MAAM,EAAE,GAAG,CAAC,CAAC,QAA0B,CAAA;QACvC,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO,KAAK,CAAA;QACnC,2EAA2E;QAC3E,IAAI,EAAE,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAA;QAC1C,OAAO,EAAE,CAAC,QAAQ,CAAC,IAAI,CACtB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CACnF,CAAA;IACF,CAAC,CAAC,CAAA;IACH,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3E,OAAO;QACN,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;QACZ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;QACb,SAAS;QACT,WAAW;QACX,OAAO,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE;QACzC,UAAU;QACV,iBAAiB,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;KAC3F,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CACvB,QAA4B,EAC5B,KAAqC,EACrC,cAAsB;IAEtB,MAAM,GAAG,GAAqB,EAAE,CAAA;IAChC,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACvE,IAAI,cAAc,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAChD,sEAAsE;IACtE,mDAAmD;IACnD,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IAC/D,yEAAyE;IACzE,uDAAuD;IACvD,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,IAAI,QAAQ,CAAC,iBAAiB,KAAK,KAAK,EAAE,CAAC;QAC1E,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;IAC9B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,KAAK,MAAM,IAAI,KAAK,CAAC,CAAC,KAAK,MAAM,CAAA;IACrD,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;SAC3B,IACJ,QAAQ,CAAC,CAAC,KAAK,IAAI;QACnB,QAAQ,CAAC,CAAC,KAAK,IAAI;QACnB,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,EACjD,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,KAAK,CAAC,MAAc;IAC5B,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,QAAQ,CAChB,OAAkC,EAClC,QAA4B,EAC5B,KAAa,EACb,KAAa,EACb,IAAc,EACd,MAAc;IAEd,MAAM,IAAI,GAAmC;QAC5C,YAAY,EACX,kFAAkF;QACnF,oBAAoB,EAAE,iDAAiD,QAAQ,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,uDAAuD;QAC/L,aAAa,EACZ,mIAAmI;QACpI,cAAc,EAAE,wCAAwC,KAAK,eAAe,KAAK,4DAA4D;QAC7I,mBAAmB,EAClB,qIAAqI;QACtI,cAAc,EACb,oIAAoI;KACrI,CAAA;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ;QAC5B,CAAC,CAAC,4CAA4C,MAAM,oBAAoB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QAC/J,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAA;IAChC,OAAO;QACN,KAAK,EAAE,MAAM;QACb,QAAQ,EAAE,uBAAuB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAmB,CAAC,EAAE;QACrE,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC3E,QAAQ,EAAE,OAAO;KACjB,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CACd,IAAc,EACd,MAAc,EACd,KAAY;IAEZ,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,GAAG,GAAG,CAAC,CAAqB,EAAE,EAAE,CACrC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAA;IACzE,KAAK,CAAC,IAAI,CACT,eAAe,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,cAAc,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,cAAc,GAAG,CACtI,CAAA;IACD,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAA;IACzD,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,GAAY,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAA;QACnF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAA;QAC7E,CAAC;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAU,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CACT,yDAAyD,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtH,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CACT,uDAAuD,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACd,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/experiment.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAUtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAsE3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA4F7D"}
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAoBtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAgG3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA8G7D"}

package/dist/eval/experiment.js

@@ -1,7 +1,9 @@
 import { EvalUsageError, validateSeed } from './bootstrap.js';
 import { buildDisclosure, describeDisclosure } from './disclosure.js';
-import { describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
+import { clarifyDeclared, describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
 import { currentEvalRunOptions } from './run-options.js';
+import { defaultSettingsSnapshot } from './settings.js';
+import { HELD_OUT_FRACTION, TASK_SPLIT_SEED, assignSplit, taskId, validateSplitFilter, } from './split.js';
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 const MAX_REPEATS = 100;
 const MAX_CASE_TIMEOUT_MS = 2_147_483_647;
@@ -30,8 +32,27 @@ export async function runExperiment(config) {
     const alike = duplicateName(config.cases.map((c) => c.name));
     if (alike !== undefined)
         throw new Error(duplicateMessage(config.name, alike));
+    // Which tasks run: every one, or the half assigned to one split. The
+    // assignment is a function of the task's identity, so it does not move.
+    const requested = validateSplitFilter(options.split);
+    const assigned = config.cases.map((c) => ({
+        evalCase: c,
+        ...assignSplit(config.name, c.name, c.split),
+    }));
+    const chosen = assigned.filter((t) => requested === 'all' || t.split === requested);
+    const cases = chosen.map((t) => t.evalCase);
+    const taskSplit = {
+        requested,
+        seed: TASK_SPLIT_SEED,
+        heldOutFraction: HELD_OUT_FRACTION,
+        tasks: chosen.map((t) => ({
+            id: taskId(config.name, t.evalCase.name),
+            split: t.split,
+            declared: t.declared,
+        })),
+    };
     // Case-major, so a case's repeats sit together in the report.
-    const total = config.cases.length * repeats;
+    const total = cases.length * repeats;
     const results = new Array(total);
     let cursor = 0;
     const worker = async () => {
@@ -39,7 +60,7 @@ export async function runExperiment(config) {
             const index = cursor++;
             if (index >= total)
                 return;
-            const evalCase = config.cases[Math.floor(index / repeats)];
+            const evalCase = cases[Math.floor(index / repeats)];
             const result = await evaluateCase(config, evalCase, threshold, timeoutMs);
             result.repeat = (index % repeats) + 1;
             if (evalCase.split)
@@ -77,7 +98,13 @@ export async function runExperiment(config) {
         intervals: stats.intervals,
         ...(repeats > 1 ? { caseSummaries: stats.caseSummaries, reliability: stats.reliability } : {}),
         splits: stats.splits,
-        disclosure: buildDisclosure(config.disclosure, { cases: config.cases.length, repeats, seed }, options, new Date().toISOString()),
+        disclosure: buildDisclosure(config.disclosure, { cases: cases.length, repeats, seed }, options, new Date().toISOString()),
+        settings: {
+            ...(options.settings ?? defaultSettingsSnapshot()),
+            read: options.settingsUse?.read ?? false,
+            readKeys: [...(options.settingsUse?.keys ?? [])].sort(),
+        },
+        taskSplit,
     };
 }
 function validateRepeats(value) {
@@ -340,11 +367,24 @@ export function formatReport(report) {
         }
     }
     if (report.splits) {
-        for (const line of describeSplits(report.splits))
-            lines.push(`  ${line}`);
+        for (const line of describeSplits(report.splits)) {
+            // The task split is the held-out one: the line about cases a suite
+            // declared as developed against would contradict the run's own header.
+            if (report.taskSplit?.requested === 'held-out' && line.startsWith('no held-out cases'))
+                continue;
+            lines.push(`  ${report.taskSplit ? clarifyDeclared(line) : line}`);
+        }
     }
     if (report.disclosure)
         lines.push(`  ${describeDisclosure(report.disclosure)}`);
+    if (report.settings) {
+        const r = report.settings;
+        lines.push(`  settings: "${r.name}", sha256 ${r.digest.slice(0, 12)}${r.read || Object.keys(r.values).length === 0 ? '' : ' (this suite does not read them)'}`);
+    }
+    if (report.taskSplit) {
+        const t = report.taskSplit;
+        lines.push(`  split: ${t.requested}, ${t.tasks.length} task${t.tasks.length === 1 ? '' : 's'} ran`);
+    }
     lines.push('');
     // Surfaced above the failures, because an inconclusive case means the
     // harness is broken and every number below it is measured over less