@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/lib/orchestrator/experiment-context.d.ts

@@ -2,14 +2,16 @@ import type { Registry } from "../core/index.js";
 import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
-export type EvaluationCorpus = {
+export type CorpusSelection = {
     kind: "agent";
     cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
-    hash: string;
 } | {
     kind: "grader";
     hash: string;
 };
+export type EvaluationCorpus = CorpusSelection & {
+    hash: string;
+};
 export type CampaignExecutionOptions = {
     config: ResolvedLuxConfig;
     registry: Registry;
@@ -25,9 +27,53 @@ export type CampaignExecutionOptions = {
     }) => void | Promise<void>;
 };
 export declare const profileOrThrow: (config: ResolvedLuxConfig, profileId: string) => {
+    environment?: {
+        imageDigest?: string | undefined;
+        toolchains?: Record<string, string> | undefined;
+        network?: "disabled" | "restricted" | "unrestricted" | undefined;
+        resources?: {
+            cpuCount?: number | undefined;
+            memoryBytes?: number | undefined;
+            diskBytes?: number | undefined;
+        } | undefined;
+    } | undefined;
     driver?: string | {
         name: string;
         config?: unknown;
+        execution?: {
+            kind: "host";
+        } | {
+            kind: "container";
+            image: string;
+            command?: string | undefined;
+            worker?: string | undefined;
+            driverModule?: string | undefined;
+            mounts?: {
+                source: string;
+                target?: string | undefined;
+                readOnly?: boolean | undefined;
+            }[] | undefined;
+            passEnv?: string[] | undefined;
+            env?: Record<string, string> | undefined;
+            homeFiles?: {
+                destination: string;
+                source: {
+                    kind: "file";
+                    path: string;
+                } | {
+                    kind: "keychain";
+                    service: string;
+                    account: string;
+                    encoding?: "go-keyring-base64" | "utf8" | undefined;
+                };
+            }[] | undefined;
+            network?: "bridge" | "none" | undefined;
+            user?: string | undefined;
+            memory?: string | undefined;
+            cpus?: number | undefined;
+            pidsLimit?: number | undefined;
+            startupTimeoutMs?: number | undefined;
+        } | undefined;
     } | undefined;
     answerer?: string | {
         name: string;
@@ -37,7 +83,9 @@ export declare const profileOrThrow: (config: ResolvedLuxConfig, profileId: stri
         provider: "claude-code" | "codex";
         model: string;
         command?: string | undefined;
-        reasoningEffort?: "high" | "low" | "medium" | "minimal" | "none" | "xhigh" | undefined;
+        inheritEnv?: true | string[] | undefined;
+        env?: Record<string, string> | undefined;
+        reasoningEffort?: "high" | "low" | "max" | "medium" | "minimal" | "none" | "ultra" | "xhigh" | undefined;
     } | undefined;
 };
 //# sourceMappingURL=experiment-context.d.ts.map

package/lib/orchestrator/experiment-context.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-context.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAEvE,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1D,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,wBAAwB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;KACxC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CAAC;AAEF,eAAO,MAAM,cAAc,WAAY,iBAAiB,aAAa,MAAM;;;;;;;;;;;;;;;CAS1E,CAAC"}
+{"version":3,"file":"experiment-context.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAGvE,MAAM,MAAM,eAAe,GACvB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;CAC3D,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAElE,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,wBAAwB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;KACxC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CAAC;AAEF,eAAO,MAAM,cAAc,WAAY,iBAAiB,aAAa,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAS1E,CAAC"}

package/lib/orchestrator/experiment-context.js

@@ -1,9 +1,10 @@
+import { resolveProfile } from "./profile.js";
 export const profileOrThrow = (config, profileId) => {
     const profile = config.profiles?.[profileId];
     if (!profile) {
         const available = Object.keys(config.profiles ?? {}).sort();
         throw new Error(`experiment profile '${profileId}' is not configured${available.length > 0 ? `; available: ${available.join(", ")}` : ""}`);
     }
-    return profile;
+    return resolveProfile(config, profile);
 };
 //# sourceMappingURL=experiment-context.js.map

package/lib/orchestrator/experiment-context.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-context.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AA+BA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAyB,EAAE,SAAiB,EAAE,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,uBAAuB,SAAS,sBAAsB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3H,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
+{"version":3,"file":"experiment-context.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AA6B9C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAyB,EAAE,SAAiB,EAAE,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,uBAAuB,SAAS,sBAAsB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3H,CAAC;IACJ,CAAC;IACD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC,CAAC"}

package/lib/orchestrator/experiment-contracts.d.ts

@@ -34,7 +34,7 @@ export declare const CampaignExecutionResultSchema: z.ZodObject<{
                 validate: z.ZodDefault<z.ZodArray<z.ZodObject<{
                     command: z.ZodString;
                     args: z.ZodDefault<z.ZodArray<z.ZodString>>;
-                    timeoutMs: z.ZodDefault<z.ZodNumber>;
+                    timeout: z.ZodDefault<z.ZodString>;
                     unsafeHostExecution: z.ZodLiteral<true>;
                 }, z.core.$strict>>>;
             }, z.core.$strict>;
@@ -74,6 +74,7 @@ export declare const CampaignExecutionResultSchema: z.ZodObject<{
             objectives: z.ZodDefault<z.ZodObject<{
                 constraints: z.ZodDefault<z.ZodArray<z.ZodObject<{
                     metric: z.ZodString;
+                    relativeTo: z.ZodExactOptional<z.ZodLiteral<"baseline">>;
                     min: z.ZodOptional<z.ZodNumber>;
                     max: z.ZodOptional<z.ZodNumber>;
                     profile: z.ZodOptional<z.ZodString>;
@@ -170,6 +171,25 @@ export declare const CampaignExecutionResultSchema: z.ZodObject<{
             passRate: z.ZodNumber;
             casePassRates: z.ZodRecord<z.ZodString, z.ZodNumber>;
             metrics: z.ZodRecord<z.ZodString, z.ZodNumber>;
+            metricDefinitions: z.ZodExactOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+                unit: z.ZodDefault<z.ZodEnum<{
+                    bytes: "bytes";
+                    count: "count";
+                    milliseconds: "milliseconds";
+                    number: "number";
+                    percent: "percent";
+                    ratio: "ratio";
+                    score: "score";
+                    tokens: "tokens";
+                    usd: "usd";
+                }>>;
+                label: z.ZodOptional<z.ZodString>;
+                better: z.ZodDefault<z.ZodEnum<{
+                    higher: "higher";
+                    lower: "lower";
+                    neutral: "neutral";
+                }>>;
+            }, z.core.$strict>>>;
             metricCalls: z.ZodExactOptional<z.ZodNumber>;
             judgeCalls: z.ZodExactOptional<z.ZodNumber>;
             unpricedModelCalls: z.ZodExactOptional<z.ZodNumber>;
@@ -181,6 +201,15 @@ export declare const CampaignExecutionResultSchema: z.ZodObject<{
             }, z.core.$strip>>;
             costUsd: z.ZodExactOptional<z.ZodNumber>;
             samples: z.ZodExactOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber>>>;
+            caseSamples: z.ZodExactOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+                identityHash: z.ZodNullable<z.ZodString>;
+                scoreBasis: z.ZodExactOptional<z.ZodNullable<z.ZodString>>;
+                clusterId: z.ZodExactOptional<z.ZodString>;
+                metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
+                    mean: z.ZodNumber;
+                    n: z.ZodNumber;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>>;
             failedCases: z.ZodArray<z.ZodObject<{
                 caseId: z.ZodString;
                 score: z.ZodNullable<z.ZodNumber>;
@@ -217,6 +246,7 @@ export declare const CampaignExecutionResultSchema: z.ZodObject<{
             constraints: z.ZodArray<z.ZodObject<{
                 candidateId: z.ZodString;
                 metric: z.ZodString;
+                relativeTo: z.ZodExactOptional<z.ZodLiteral<"baseline">>;
                 profile: z.ZodExactOptional<z.ZodString>;
                 split: z.ZodEnum<{
                     test: "test";

package/lib/orchestrator/experiment-contracts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-contracts.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA4B9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}
+{"version":3,"file":"experiment-contracts.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA4B9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}

package/lib/orchestrator/experiment-corpus.d.ts

@@ -1,6 +1,6 @@
 import { type Candidate, type ExperimentCampaign, type ExperimentSelector } from "../core/index.js";
 import { discoverCases } from "./discover.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import type { CampaignExecutionOptions, CorpusSelection, EvaluationCorpus } from "./experiment-context.js";
 import { loadEvalCase } from "./loader.js";
 type ExperimentSplit = "train" | "validation" | "test";
 export declare const selectorsBySplit: (campaign: ExperimentCampaign) => Array<{
@@ -16,6 +16,7 @@ export declare const caseRubricCliReferences: (execution: CampaignExecutionOptio
 export declare const agentCorpusHash: (execution: CampaignExecutionOptions, cases: Awaited<ReturnType<typeof discoverCases>>["cases"], splits: ReturnType<typeof selectorsBySplit>) => Promise<string>;
 export declare const graderCorpusHash: (execution: CampaignExecutionOptions, splits: ReturnType<typeof selectorsBySplit>) => Promise<string>;
 export declare const prepareCorpus: (execution: CampaignExecutionOptions) => Promise<EvaluationCorpus>;
+export declare const inspectCorpus: (execution: CampaignExecutionOptions) => Promise<CorpusSelection>;
 export declare const assertGraderSubject: (execution: CampaignExecutionOptions, baseline: Candidate) => Promise<void>;
 export {};
 //# sourceMappingURL=experiment-corpus.d.ts.map

package/lib/orchestrator/experiment-corpus.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-corpus.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,SAAS,EAEd,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAExB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAW1F,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAK3C,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAEvD,eAAO,MAAM,gBAAgB,aACjB,kBAAkB,KAC3B,KAAK,CAAC;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAA;CAAE,CAI/D,CAAC;AA0BF,eAAO,MAAM,uBAAuB,cACvB,wBAAwB,YACzB,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,KACjD,KAAK,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;CACjB,CAsBA,CAAC;AAoBF,eAAO,MAAM,eAAe,cACf,wBAAwB,SAC5B,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,UACjD,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CAsEhB,CAAC;AAEF,eAAO,MAAM,gBAAgB,cAChB,wBAAwB,UAC3B,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CAgFhB,CAAC;AAEF,eAAO,MAAM,aAAa,cACb,wBAAwB,KAClC,OAAO,CAAC,gBAAgB,CA6E1B,CAAC;AAEF,eAAO,MAAM,mBAAmB,cACnB,wBAAwB,YACzB,SAAS,KAClB,OAAO,CAAC,IAAI,CAyDd,CAAC"}
+{"version":3,"file":"experiment-corpus.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAIA,OAAO,EAEL,KAAK,SAAS,EAEd,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAExB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAE3D,OAAO,KAAK,EACV,wBAAwB,EACxB,eAAe,EACf,gBAAgB,EACjB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAK3C,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAEvD,eAAO,MAAM,gBAAgB,aACjB,kBAAkB,KAC3B,KAAK,CAAC;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAA;CAAE,CAI/D,CAAC;AA0BF,eAAO,MAAM,uBAAuB,cACvB,wBAAwB,YACzB,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,KACjD,KAAK,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;CACjB,CAsBA,CAAC;AAoBF,eAAO,MAAM,eAAe,cACf,wBAAwB,SAC5B,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,UACjD,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CA4DhB,CAAC;AAEF,eAAO,MAAM,gBAAgB,cAChB,wBAAwB,UAC3B,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CAgFhB,CAAC;AAEF,eAAO,MAAM,aAAa,cACb,wBAAwB,KAClC,OAAO,CAAC,gBAAgB,CAW1B,CAAC;AAEF,eAAO,MAAM,aAAa,cACb,wBAAwB,KAClC,OAAO,CAAC,eAAe,CA8FzB,CAAC;AAEF,eAAO,MAAM,mBAAmB,cACnB,wBAAwB,YACzB,SAAS,KAClB,OAAO,CAAC,IAAI,CAyDd,CAAC"}

package/lib/orchestrator/experiment-corpus.js

@@ -1,11 +1,13 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import { bindSubjectRoot, contentHash, valueHash, } from "../core/index.js";
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
+import { experimentDriverIdentity } from "./driver-identity.js";
 import { profileOrThrow } from "./experiment-context.js";
-import { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes, referencedAnswererCli, referencedCli, } from "./experiment-runtime-identity.js";
+import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
 import { loadEvalCase } from "./loader.js";
 import { resolveAnswererReference, resolveDriverReference } from "./orchestrator.js";
@@ -59,36 +61,19 @@ export const agentCorpusHash = async (execution, cases, splits) => {
             .flatMap((split) => selectCases(cases, split.selector))
             .map((evalCase) => [evalCase.path, evalCase])).values(),
     ];
-    const caseBehaviorHashes = new Map();
-    const configuredSourceHashes = new Map();
     const definitions = await Promise.all(selected.map(async (evalCase) => {
         const authored = await loadEvalCase(evalCase.path);
         const bound = bindSubjectRoot(authored, execution.loaded.subjectRoot);
         const profileCliIdentities = await Promise.all(execution.loaded.campaign.profiles.map(async (profileId) => {
             const profile = profileOrThrow(execution.config, profileId);
             const harness = profile.harness ?? execution.config.harness;
-            const driver = resolveDriverReference(bound, profile.driver ?? execution.config.defaultDriver);
-            const answerer = resolveAnswererReference(bound, profile.answerer ?? execution.config.defaultAnswerer, harness);
-            const driverCli = referencedCli(driver);
+            const driver = bindSubjectRoot(resolveDriverReference(bound, profile.driver ?? execution.config.defaultDriver, undefined, execution.config.driverDefaults), execution.loaded.subjectRoot);
+            const answerer = bindSubjectRoot(resolveAnswererReference(bound, profile.answerer ?? execution.config.defaultAnswerer, harness), execution.loaded.subjectRoot);
             const answererCli = referencedAnswererCli(answerer);
-            const configuredRoot = driverCli?.name === "claude-code" ? claudeConfigRoot(driver) : null;
-            let driverClaudeBehaviorHash = null;
-            if (configuredRoot) {
-                let hash = caseBehaviorHashes.get(configuredRoot);
-                if (!hash) {
-                    hash = claudeUserBehaviorHash(configuredRoot);
-                    caseBehaviorHashes.set(configuredRoot, hash);
-                }
-                driverClaudeBehaviorHash = await hash;
-            }
             return {
                 profileId,
                 driver,
-                driverCliIdentity: driverCli
-                    ? await cliIdentity(driverCli.command, execution.config.rootDir)
-                    : null,
-                driverClaudeBehaviorHash,
-                configuredDriverSources: await configuredClaudeSourceHashes(driver, execution.config.rootDir, configuredSourceHashes),
+                driverIdentity: await experimentDriverIdentity(driver, execution.registry, execution.config.rootDir),
                 answerer,
                 answererCliIdentity: answererCli
                     ? await cliIdentity(answererCli.command, execution.config.rootDir)
@@ -175,6 +160,15 @@ export const graderCorpusHash = async (execution, splits) => {
     return valueHash({ annotationsHash: loaded.hash, evidence });
 };
 export const prepareCorpus = async (execution) => {
+    const corpus = await inspectCorpus(execution);
+    if (corpus.kind === "grader")
+        return corpus;
+    return {
+        ...corpus,
+        hash: await agentCorpusHash(execution, corpus.cases, selectorsBySplit(execution.loaded.campaign)),
+    };
+};
+export const inspectCorpus = async (execution) => {
     const campaign = execution.loaded.campaign;
     const splits = selectorsBySplit(campaign);
     if (campaign.evaluation.kind === "grader") {
@@ -198,6 +192,19 @@ export const prepareCorpus = async (execution) => {
         throw new Error(`experiment case discovery failed: ${discovery.problems.map((problem) => `${problem.path}: ${problem.reason}`).join("; ")}`);
     }
     const owners = new Map();
+    assertDisjointClusters(splits.map((split) => ({
+        name: split.name,
+        items: selectCases(discovery.cases, split.selector),
+    })));
+    for (const constraint of campaign.objectives.constraints) {
+        if (constraint.confidence?.method !== "wilson")
+            continue;
+        const selector = splits.find((split) => split.name === constraint.split)?.selector;
+        if (selector &&
+            selectCases(discovery.cases, selector).some((evalCase) => evalCase.cluster !== undefined)) {
+            throw new Error("authored case clusters produce bounded means; use Hoeffding or percentile-bootstrap confidence instead of Wilson");
+        }
+    }
     const caseIds = new Map();
     const evidenceOwners = new Map();
     for (const split of splits) {
@@ -234,7 +241,6 @@ export const prepareCorpus = async (execution) => {
     return {
         kind: "agent",
         cases: discovery.cases,
-        hash: await agentCorpusHash(execution, discovery.cases, splits),
     };
 };
 export const assertGraderSubject = async (execution, baseline) => {

package/lib/orchestrator/experiment-corpus.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-corpus.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EACL,eAAe,EAEf,WAAW,EAGX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,8BAA8B,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE3D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,WAAW,EACX,4BAA4B,EAC5B,qBAAqB,EACrB,aAAa,GACd,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAI1D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,QAA4B,EACoC,EAAE,CAAC;IACnE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/F,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;IAC5D,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;CAC7F,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAC/B,SAAmC,EACnC,QAAgB,EAChB,QAA8D,EACtC,EAAE;IAC1B,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,OAAO,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpG,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EACjC,SAAmC,EACnC,QAAgB,EACC,EAAE;IACnB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC;IACtF,MAAM,oBAAoB,GAAG,UAAU,CAAC,GAAG,CACzC,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CACxE,CAAC;IACF,OAAO,SAAS,CAAC;QACf,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,oBAAoB,EAAE;QACvD,QAAQ,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC;KACnE,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAAmC,EACnC,QAAkD,EAMjD,EAAE;IACH,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CACvE,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;IACF,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;QAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;QAC5D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,EAAE,QAAQ,IAAI,aAAa,CAAC;YACvE,MAAM,eAAe,GACnB,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,QAAQ,CAAC;YACzE,OAAO;gBACL,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,aAAa,KAAK,EAAE;gBAC9C,SAAS;gBACT,QAAQ;gBACR,OAAO,EACL,MAAM,CAAC,OAAO;oBACd,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;oBAChD,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;aAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAC9B,SAAmC,EACnC,QAAkD,EAQlD,EAAE,CACF,OAAO,CAAC,GAAG,CACT,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IACrF,GAAG,SAAS;IACZ,WAAW,EAAE,MAAM,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;CAClE,CAAC,CAAC,CACJ,CAAC;AAEJ,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,SAAmC,EACnC,KAAyD,EACzD,MAA2C,EAC1B,EAAE;IACnB,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAkC,CAAC;IACrE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACtE,MAAM,oBAAoB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;YACzD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5D,MAAM,MAAM,GAAG,sBAAsB,CACnC,KAAK,EACL,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CACjD,CAAC;YACF,MAAM,QAAQ,GAAG,wBAAwB,CACvC,KAAK,EACL,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,EACpD,OAAO,CACR,CAAC;YACF,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,WAAW,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACpD,MAAM,cAAc,GAClB,SAAS,EAAE,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACtE,IAAI,wBAAwB,GAAkB,IAAI,CAAC;YACnD,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,IAAI,GAAG,sBAAsB,CAAC,cAAc,CAAC,CAAC;oBAC9C,kBAAkB,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;gBAC/C,CAAC;gBACD,wBAAwB,GAAG,MAAM,IAAI,CAAC;YACxC,CAAC;YACD,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,iBAAiB,EAAE,SAAS;oBAC1B,CAAC,CAAC,MAAM,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAChE,CAAC,CAAC,IAAI;gBACR,wBAAwB;gBACxB,uBAAuB,EAAE,MAAM,4BAA4B,CACzD,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,OAAO,EACxB,sBAAsB,CACvB;gBACD,QAAQ;gBACR,mBAAmB,EAAE,WAAW;oBAC9B,CAAC,CAAC,MAAM,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAClE,CAAC,CAAC,IAAI;aACT,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QACF,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,UAAU,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;YACpF,oBAAoB;YACpB,uBAAuB,EAAE,MAAM,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IACF,OAAO,SAAS,CAAC,WAAW,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,SAAmC,EACnC,MAA2C,EAC1B,EAAE;IACnB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC/E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CAC1D,CAAC;YACF,IAAI,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CACrG,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBACnE,IAAI,CAAC;oBACH,OAAO;wBACL,GAAG,QAAQ;wBACX,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;qBACpE,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACvD,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;oBAC7D,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YACF,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,EAAE;gBAC3B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,SAAS,CAAC;oBACtB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;oBACnB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;oBACvB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;wBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;4BAAE,OAAO,EAAE,CAAC;wBACpE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC;wBAC7C,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC;oBACF,SAAS;iBACV,CAAC;gBACF,MAAM;aACP,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAC7D,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAU,CACrD,CACF,CACF,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAA4D,CAAC;IAC5F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;YAC7F,MAAM,IAAI,KAAK,CACb,UAAU,KAAK,kBAAkB,KAAK,CAAC,YAAY,MAAM,KAAK,CAAC,KAAK,UAAU,IAAI,CAAC,YAAY,MAAM,KAAK,qCAAqC,CAChJ,CAAC;QACJ,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,SAAS,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,SAAmC,EACR,EAAE;IAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACpF,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC;SACzE,CAAC,CAAC,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CACb,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,yCAAyC,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,8BAA8B,CAAC,cAAc,CAAC,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IAC7E,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,SAAS,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CACtE,CAAC;IACF,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,qCAAqC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5H,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,GAAG,EAG3B,CAAC;IACJ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,4BAA4B,CAAC,CAAC;QACzF,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,YAAY,IAAI,YAAY,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,uBAAuB,QAAQ,CAAC,EAAE,sBAAsB,YAAY,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAC5F,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACb,cAAc,QAAQ,CAAC,EAAE,qBAAqB,KAAK,QAAQ,KAAK,CAAC,IAAI,SAAS,CAC/E,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,YAAY,GAAG,MAAM,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACvD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,KAAK,GACT,aAAa,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI;oBAChC,CAAC,CAAC,6BAA6B,KAAK,CAAC,IAAI,EAAE;oBAC3C,CAAC,CAAC,eAAe,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,SAAS,KAAK,YAAY,aAAa,CAAC,MAAM,MAAM,aAAa,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,KAAK,CAAC,IAAI,yCAAyC,CACtJ,CAAC;YACJ,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,KAAK,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,IAAI,EAAE,MAAM,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;KAChE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,SAAmC,EACnC,QAAmB,EACJ,EAAE;IACjB,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IACnE,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,IAAI,EAAE,CAAC;IAChF,MAAM,WAAW,GAAG;QAClB,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC9E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChG,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IACE,WAAW,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM;QAC5C,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,cAAc,CAAC,KAAK,CAAC,CAAC,EACjE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;QAC1F,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;QAC1F,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,EAAE,CACjF,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAC9E,CAAC;QACF,MAAM,mBAAmB,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,CACnD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC1D,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,yCAAyC,UAAU,+DAA+D,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnJ,CAAC;QACJ,CAAC;QACD,MAAM,mBAAmB,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,MAAM,CACtD,CAAC,EAAE,EAAE,EAAE,CACL,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CACrF,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,oCAAoC,UAAU,mEAAmE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClJ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CACpC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAC/D,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,+EAA+E,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/G,CAAC;IACJ,CAAC;AACH,CAAC,CAAC"}
+{"version":3,"file":"experiment-corpus.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EACL,eAAe,EAEf,WAAW,EAGX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,8BAA8B,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAMhE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAI1D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,QAA4B,EACoC,EAAE,CAAC;IACnE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/F,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;IAC5D,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;CAC7F,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAC/B,SAAmC,EACnC,QAAgB,EAChB,QAA8D,EACtC,EAAE;IAC1B,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,OAAO,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpG,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EACjC,SAAmC,EACnC,QAAgB,EACC,EAAE;IACnB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC;IACtF,MAAM,oBAAoB,GAAG,UAAU,CAAC,GAAG,CACzC,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CACxE,CAAC;IACF,OAAO,SAAS,CAAC;QACf,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,oBAAoB,EAAE;QACvD,QAAQ,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC;KACnE,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAAmC,EACnC,QAAkD,EAMjD,EAAE;IACH,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CACvE,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;IACF,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;QAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;QAC5D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,EAAE,QAAQ,IAAI,aAAa,CAAC;YACvE,MAAM,eAAe,GACnB,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,QAAQ,CAAC;YACzE,OAAO;gBACL,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,aAAa,KAAK,EAAE;gBAC9C,SAAS;gBACT,QAAQ;gBACR,OAAO,EACL,MAAM,CAAC,OAAO;oBACd,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;oBAChD,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;aAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAC9B,SAAmC,EACnC,QAAkD,EAQlD,EAAE,CACF,OAAO,CAAC,GAAG,CACT,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IACrF,GAAG,SAAS;IACZ,WAAW,EAAE,MAAM,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;CAClE,CAAC,CAAC,CACJ,CAAC;AAEJ,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,SAAmC,EACnC,KAAyD,EACzD,MAA2C,EAC1B,EAAE;IACnB,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACtE,MAAM,oBAAoB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;YACzD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5D,MAAM,MAAM,GAAG,eAAe,CAC5B,sBAAsB,CACpB,KAAK,EACL,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,EAChD,SAAS,EACT,SAAS,CAAC,MAAM,CAAC,cAAc,CAChC,EACD,SAAS,CAAC,MAAM,CAAC,WAAW,CAC7B,CAAC;YACF,MAAM,QAAQ,GAAG,eAAe,CAC9B,wBAAwB,CACtB,KAAK,EACL,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,EACpD,OAAO,CACR,EACD,SAAS,CAAC,MAAM,CAAC,WAAW,CAC7B,CAAC;YACF,MAAM,WAAW,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACpD,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,cAAc,EAAE,MAAM,wBAAwB,CAC5C,MAAM,EACN,SAAS,CAAC,QAAQ,EAClB,SAAS,CAAC,MAAM,CAAC,OAAO,CACzB;gBACD,QAAQ;gBACR,mBAAmB,EAAE,WAAW;oBAC9B,CAAC,CAAC,MAAM,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAClE,CAAC,CAAC,IAAI;aACT,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QACF,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,UAAU,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;YACpF,oBAAoB;YACpB,uBAAuB,EAAE,MAAM,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IACF,OAAO,SAAS,CAAC,WAAW,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,SAAmC,EACnC,MAA2C,EAC1B,EAAE;IACnB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC/E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CAC1D,CAAC;YACF,IAAI,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CACrG,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBACnE,IAAI,CAAC;oBACH,OAAO;wBACL,GAAG,QAAQ;wBACX,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;qBACpE,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACvD,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;oBAC7D,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YACF,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,EAAE;gBAC3B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,SAAS,CAAC;oBACtB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;oBACnB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;oBACvB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;wBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;4BAAE,OAAO,EAAE,CAAC;wBACpE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC;wBAC7C,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC;oBACF,SAAS;iBACV,CAAC;gBACF,MAAM;aACP,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAC7D,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAU,CACrD,CACF,CACF,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAA4D,CAAC;IAC5F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;YAC7F,MAAM,IAAI,KAAK,CACb,UAAU,KAAK,kBAAkB,KAAK,CAAC,YAAY,MAAM,KAAK,CAAC,KAAK,UAAU,IAAI,CAAC,YAAY,MAAM,KAAK,qCAAqC,CAChJ,CAAC;QACJ,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,SAAS,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,SAAmC,EACR,EAAE;IAC7B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC5C,OAAO;QACL,GAAG,MAAM;QACT,IAAI,EAAE,MAAM,eAAe,CACzB,SAAS,EACT,MAAM,CAAC,KAAK,EACZ,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAC5C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,SAAmC,EACT,EAAE;IAC5B,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACpF,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC;SACzE,CAAC,CAAC,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CACb,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,yCAAyC,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,8BAA8B,CAAC,cAAc,CAAC,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IAC7E,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,SAAS,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CACtE,CAAC;IACF,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,qCAAqC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5H,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,sBAAsB,CACpB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC;KACpD,CAAC,CAAC,CACJ,CAAC;IACF,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;QACzD,IAAI,UAAU,CAAC,UAAU,EAAE,MAAM,KAAK,QAAQ;YAAE,SAAS;QACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC;QACnF,IACE,QAAQ;YACR,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,EACzF,CAAC;YACD,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,GAAG,EAG3B,CAAC;IACJ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,4BAA4B,CAAC,CAAC;QACzF,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,YAAY,IAAI,YAAY,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,uBAAuB,QAAQ,CAAC,EAAE,sBAAsB,YAAY,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAC5F,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACb,cAAc,QAAQ,CAAC,EAAE,qBAAqB,KAAK,QAAQ,KAAK,CAAC,IAAI,SAAS,CAC/E,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,YAAY,GAAG,MAAM,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACvD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,KAAK,GACT,aAAa,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI;oBAChC,CAAC,CAAC,6BAA6B,KAAK,CAAC,IAAI,EAAE;oBAC3C,CAAC,CAAC,eAAe,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,SAAS,KAAK,YAAY,aAAa,CAAC,MAAM,MAAM,aAAa,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,KAAK,CAAC,IAAI,yCAAyC,CACtJ,CAAC;YACJ,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,KAAK,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,SAAS,CAAC,KAAK;KACvB,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,SAAmC,EACnC,QAAmB,EACJ,EAAE;IACjB,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IACnE,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,IAAI,EAAE,CAAC;IAChF,MAAM,WAAW,GAAG;QAClB,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC9E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChG,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IACE,WAAW,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM;QAC5C,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,cAAc,CAAC,KAAK,CAAC,CAAC,EACjE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;QAC1F,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;QAC1F,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,EAAE,CACjF,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAC9E,CAAC;QACF,MAAM,mBAAmB,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,CACnD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC1D,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,yCAAyC,UAAU,+DAA+D,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnJ,CAAC;QACJ,CAAC;QACD,MAAM,mBAAmB,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,MAAM,CACtD,CAAC,EAAE,EAAE,EAAE,CACL,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CACrF,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,oCAAoC,UAAU,mEAAmE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClJ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CACpC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAC/D,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,+EAA+E,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/G,CAAC;IACJ,CAAC;AACH,CAAC,CAAC"}

package/lib/orchestrator/experiment-effects.d.ts

@@ -0,0 +1,33 @@
+import { z } from "zod";
+import type { ExperimentRun } from "./experiment-runs.js";
+export declare const ExperimentEffectsSchema: z.ZodArray<z.ZodObject<{
+    caseIds: z.ZodArray<z.ZodString>;
+    n: z.ZodNumber;
+    caseCount: z.ZodOptional<z.ZodNumber>;
+    clusters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+    meanDelta: z.ZodNullable<z.ZodNumber>;
+    interval: z.ZodNullable<z.ZodObject<{
+        method: z.ZodEnum<{
+            hoeffding: "hoeffding";
+            "percentile-bootstrap": "percentile-bootstrap";
+            wilson: "wilson";
+        }>;
+        level: z.ZodNumber;
+        estimate: z.ZodNumber;
+        lower: z.ZodNumber;
+        upper: z.ZodNumber;
+        n: z.ZodNumber;
+        resamples: z.ZodNumber;
+    }, z.core.$strict>>;
+    note: z.ZodNullable<z.ZodString>;
+    candidateId: z.ZodString;
+    profileId: z.ZodString;
+    split: z.ZodEnum<{
+        test: "test";
+        train: "train";
+        validation: "validation";
+    }>;
+    metric: z.ZodString;
+}, z.core.$strip>>;
+export declare const experimentEffects: (experiment: ExperimentRun) => z.infer<typeof ExperimentEffectsSchema>;
+//# sourceMappingURL=experiment-effects.d.ts.map

package/lib/orchestrator/experiment-effects.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-effects.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-effects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAI1D,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAOnC,CAAC;AAEF,eAAO,MAAM,iBAAiB,eAChB,aAAa,KACxB,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CA6CxC,CAAC"}

package/lib/orchestrator/experiment-effects.js

@@ -0,0 +1,46 @@
+import { z } from "zod";
+import { ConfidenceConfigSchema } from "../core/index.js";
+import { candidateEvaluations } from "./experiment-selection.js";
+import { PairedEffectSchema, pairedEffect } from "./paired-effects.js";
+export const ExperimentEffectsSchema = z.array(PairedEffectSchema.extend({
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    metric: z.string(),
+}));
+export const experimentEffects = (experiment) => {
+    const candidateIds = experiment.selection?.decisionCandidateIds ??
+        experiment.candidates.map((candidate) => candidate.id);
+    const baseline = candidateEvaluations(experiment, "baseline");
+    const metrics = new Set([
+        "quality.passRate",
+        experiment.evaluationKind === "grader" ? "harness.total" : "agent.total",
+        ...(experiment.evaluationKind === "agent" ? ["score", "interrupts.ask", "toolCalls"] : []),
+        ...experiment.campaignSnapshot.objectives.constraints
+            .filter((constraint) => constraint.relativeTo)
+            .map((constraint) => constraint.metric),
+    ]);
+    return candidateIds
+        .filter((id) => id !== "baseline")
+        .flatMap((candidateId) => candidateEvaluations(experiment, candidateId).flatMap((after) => [...metrics].map((metric) => {
+        const before = baseline.find((entry) => entry.profileId === after.profileId && entry.split === after.split);
+        const constraint = experiment.campaignSnapshot.objectives.constraints.find((entry) => entry.relativeTo &&
+            entry.metric === metric &&
+            entry.split === after.split &&
+            (!entry.profile || entry.profile === after.profileId));
+        const confidence = constraint?.confidence ??
+            ConfidenceConfigSchema.parse({
+                method: ["quality.passRate", "score"].includes(metric)
+                    ? "hoeffding"
+                    : "percentile-bootstrap",
+            });
+        return {
+            candidateId,
+            profileId: after.profileId,
+            split: after.split,
+            metric,
+            ...pairedEffect(before, after, metric, confidence),
+        };
+    })));
+};
+//# sourceMappingURL=experiment-effects.js.map

package/lib/orchestrator/experiment-effects.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-effects.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-effects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEvE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC5C,kBAAkB,CAAC,MAAM,CAAC;IACxB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CACH,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,UAAyB,EACgB,EAAE;IAC3C,MAAM,YAAY,GAChB,UAAU,CAAC,SAAS,EAAE,oBAAoB;QAC1C,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,kBAAkB;QAClB,UAAU,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa;QACxE,GAAG,CAAC,UAAU,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,WAAW;aAClD,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;aAC7C,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;KAC1C,CAAC,CAAC;IACH,OAAO,YAAY;SAChB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,UAAU,CAAC;SACjC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACvB,oBAAoB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9D,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAC1B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAC9E,CAAC;QACF,MAAM,UAAU,GAAG,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CACxE,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,MAAM,KAAK,MAAM;YACvB,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;YAC3B,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,SAAS,CAAC,CACxD,CAAC;QACF,MAAM,UAAU,GACd,UAAU,EAAE,UAAU;YACtB,sBAAsB,CAAC,KAAK,CAAC;gBAC3B,MAAM,EAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACpD,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,sBAAsB;aAC3B,CAAC,CAAC;QACL,OAAO;YACL,WAAW;YACX,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM;YACN,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC;SACnD,CAAC;IACJ,CAAC,CAAC,CACH,CACF,CAAC;AACN,CAAC,CAAC"}

package/lib/orchestrator/experiment-evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-evaluation.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-evaluation.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAEL,KAAK,SAAS,EAGd,KAAK,kBAAkB,EAIxB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,KAAK,aAAa,EAAe,MAAM,eAAe,CAAC;AAEhE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAOtF,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAuFvD,QAAA,MAAM,0BAA0B;;;;;;;;;;;kBAQrB,CAAC;AACZ,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAkQ9E,eAAO,MAAM,mCAAmC,GAAU,CAAC,2BAChC,MAAM,OAAO,CAAC,IAAI,CAAC,cAChC,MAAM,OAAO,CAAC,CAAC,CAAC,KAC3B,OAAO,CAAC,CAAC,CAOX,CAAC;AAYF,eAAO,MAAM,sBAAsB,YAAmB;IACpD,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CAgP/B,CAAC;AAEF,eAAO,MAAM,8BAA8B,YAAmB;IAC5D,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CA+E/B,CAAC"}
+{"version":3,"file":"experiment-evaluation.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-evaluation.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,KAAK,SAAS,EAGd,KAAK,kBAAkB,EAIxB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,KAAK,aAAa,EAAe,MAAM,eAAe,CAAC;AAEhE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAStF,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAyGvD,QAAA,MAAM,0BAA0B;;;;;;;;;;;kBAQrB,CAAC;AACZ,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AA0T9E,eAAO,MAAM,mCAAmC,GAAU,CAAC,2BAChC,MAAM,OAAO,CAAC,IAAI,CAAC,cAChC,MAAM,OAAO,CAAC,CAAC,CAAC,KAC3B,OAAO,CAAC,CAAC,CAOX,CAAC;AAYF,eAAO,MAAM,sBAAsB,YAAmB;IACpD,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CAoP/B,CAAC;AAEF,eAAO,MAAM,8BAA8B,YAAmB;IAC5D,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CA+E/B,CAAC"}