@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment-budget.d.ts

@@ -0,0 +1,11 @@
+import { type ExperimentCampaign } from "../core/index.js";
+import type { CampaignExecutionOptions } from "./experiment-context.js";
+import { type OpenedExperimentRun } from "./experiment-runs.js";
+export declare const throwIfAborted: (signal: AbortSignal | undefined) => void;
+export declare const boundedExecutionSignal: (execution: CampaignExecutionOptions, experiment: OpenedExperimentRun) => AbortSignal;
+export declare const ensureBudget: (campaign: ExperimentCampaign, experiment: OpenedExperimentRun, additional?: {
+    metricCalls?: number;
+    judgeCalls?: number;
+    proposalCalls?: number;
+}) => void;
+//# sourceMappingURL=experiment-budget.d.ts.map

package/lib/orchestrator/experiment-budget.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-budget.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-budget.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAe,MAAM,kBAAkB,CAAC;AACxE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3F,eAAO,MAAM,cAAc,WAAY,WAAW,GAAG,SAAS,KAAG,IAGhE,CAAC;AAEF,eAAO,MAAM,sBAAsB,cACtB,wBAAwB,cACvB,mBAAmB,KAC9B,WAUF,CAAC;AAEF,eAAO,MAAM,YAAY,aACb,kBAAkB,cAChB,mBAAmB,eACnB;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,KAChF,IAgCF,CAAC"}

package/lib/orchestrator/experiment-budget.js

@@ -0,0 +1,51 @@
+import { totalTokens } from "../core/index.js";
+import { experimentActiveElapsedMs } from "./experiment-runs.js";
+export const throwIfAborted = (signal) => {
+    if (!signal?.aborted)
+        return;
+    throw signal.reason instanceof Error ? signal.reason : new Error("experiment aborted");
+};
+export const boundedExecutionSignal = (execution, experiment) => {
+    const elapsed = experimentActiveElapsedMs(experiment.manifest);
+    const remaining = execution.loaded.campaign.budget.maxWallTimeMs - elapsed;
+    if (remaining <= 0) {
+        throw new Error(`experiment exceeded its wall-time budget ${execution.loaded.campaign.budget.maxWallTimeMs}ms`);
+    }
+    const deadline = AbortSignal.timeout(Math.max(1, remaining));
+    return execution.abortSignal ? AbortSignal.any([execution.abortSignal, deadline]) : deadline;
+};
+export const ensureBudget = (campaign, experiment, additional = {}) => {
+    const usage = experiment.manifest.budgetUsage;
+    const elapsed = experimentActiveElapsedMs(experiment.manifest);
+    const checks = [
+        [
+            usage.metricCalls + (additional.metricCalls ?? 0) > campaign.budget.maxMetricCalls,
+            `metric-call budget ${campaign.budget.maxMetricCalls}`,
+        ],
+        [
+            usage.judgeCalls + (additional.judgeCalls ?? 0) > campaign.budget.maxJudgeCalls,
+            `judge-call budget ${campaign.budget.maxJudgeCalls}`,
+        ],
+        [
+            usage.proposalCalls + (additional.proposalCalls ?? 0) > campaign.budget.maxProposalCalls,
+            `proposal-call budget ${campaign.budget.maxProposalCalls}`,
+        ],
+        [
+            totalTokens(usage.usage) > campaign.budget.maxTokens,
+            `token budget ${campaign.budget.maxTokens}`,
+        ],
+        [usage.costUsd > campaign.budget.maxCostUsd, `cost budget $${campaign.budget.maxCostUsd}`],
+        [
+            (usage.unpricedModelCalls ?? 0) > campaign.budget.maxUnpricedModelCalls,
+            `unpriced-model-call budget ${campaign.budget.maxUnpricedModelCalls}`,
+        ],
+        [
+            elapsed > campaign.budget.maxWallTimeMs,
+            `wall-time budget ${campaign.budget.maxWallTimeMs}ms`,
+        ],
+    ];
+    const exceeded = checks.find(([failed]) => failed);
+    if (exceeded)
+        throw new Error(`experiment exceeded its ${exceeded[1]}`);
+};
+//# sourceMappingURL=experiment-budget.js.map

package/lib/orchestrator/experiment-budget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-budget.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-budget.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAExE,OAAO,EAAE,yBAAyB,EAA4B,MAAM,sBAAsB,CAAC;AAE3F,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAQ,EAAE;IACtE,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,SAAmC,EACnC,UAA+B,EAClB,EAAE;IACf,MAAM,OAAO,GAAG,yBAAyB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC;IAC3E,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,4CAA4C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,IAAI,CAC/F,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,QAA4B,EAC5B,UAA+B,EAC/B,UAAU,GAA0E,EAAE,EAChF,EAAE;IACR,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;IAC9C,MAAM,OAAO,GAAG,yBAAyB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAwB;QAClC;YACE,KAAK,CAAC,WAAW,GAAG,CAAC,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,cAAc;YAClF,sBAAsB,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE;SACvD;QACD;YACE,KAAK,CAAC,UAAU,GAAG,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa;YAC/E,qBAAqB,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE;SACrD;QACD;YACE,KAAK,CAAC,aAAa,GAAG,CAAC,UAAU,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,gBAAgB;YACxF,wBAAwB,QAAQ,CAAC,MAAM,CAAC,gBAAgB,EAAE;SAC3D;QACD;YACE,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS;YACpD,gBAAgB,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE;SAC5C;QACD,CAAC,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,gBAAgB,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1F;YACE,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,qBAAqB;YACvE,8BAA8B,QAAQ,CAAC,MAAM,CAAC,qBAAqB,EAAE;SACtE;QACD;YACE,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa;YACvC,oBAAoB,QAAQ,CAAC,MAAM,CAAC,aAAa,IAAI;SACtD;KACF,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC1E,CAAC,CAAC"}

package/lib/orchestrator/experiment-context.d.ts

@@ -0,0 +1,43 @@
+import type { Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+import type { discoverCases } from "./discover.js";
+import type { LoadedExperimentCampaign } from "./experiment-loader.js";
+export type EvaluationCorpus = {
+    kind: "agent";
+    cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
+    hash: string;
+} | {
+    kind: "grader";
+    hash: string;
+};
+export type CampaignExecutionOptions = {
+    config: ResolvedLuxConfig;
+    registry: Registry;
+    loaded: LoadedExperimentCampaign;
+    runsRoot?: string;
+    resumeFrom?: string;
+    abortSignal?: AbortSignal;
+    onEvaluationUnit?: (unit: {
+        unitId: string;
+        candidateId: string;
+        profileId: string;
+        split: "train" | "validation" | "test";
+    }) => void | Promise<void>;
+};
+export declare const profileOrThrow: (config: ResolvedLuxConfig, profileId: string) => {
+    driver?: string | {
+        name: string;
+        config?: unknown;
+    } | undefined;
+    answerer?: string | {
+        name: string;
+        config?: unknown;
+    } | undefined;
+    harness?: {
+        provider: "claude-code" | "codex";
+        model: string;
+        command?: string | undefined;
+        reasoningEffort?: "high" | "low" | "medium" | "minimal" | "none" | "xhigh" | undefined;
+    } | undefined;
+};
+//# sourceMappingURL=experiment-context.d.ts.map

package/lib/orchestrator/experiment-context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-context.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAEvE,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1D,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,wBAAwB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;KACxC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CAAC;AAEF,eAAO,MAAM,cAAc,WAAY,iBAAiB,aAAa,MAAM;;;;;;;;;;;;;;;CAS1E,CAAC"}

package/lib/orchestrator/experiment-context.js

@@ -0,0 +1,9 @@
+export const profileOrThrow = (config, profileId) => {
+    const profile = config.profiles?.[profileId];
+    if (!profile) {
+        const available = Object.keys(config.profiles ?? {}).sort();
+        throw new Error(`experiment profile '${profileId}' is not configured${available.length > 0 ? `; available: ${available.join(", ")}` : ""}`);
+    }
+    return profile;
+};
+//# sourceMappingURL=experiment-context.js.map

package/lib/orchestrator/experiment-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-context.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-context.ts"],"names":[],"mappings":"AA+BA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAyB,EAAE,SAAiB,EAAE,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,uBAAuB,SAAS,sBAAsB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3H,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/lib/orchestrator/experiment-contracts.d.ts

@@ -0,0 +1,290 @@
+import { z } from "zod";
+export declare const CampaignExecutionResultSchema: z.ZodObject<{
+    experimentDir: z.ZodString;
+    champion: z.ZodNullable<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        id: z.ZodString;
+        parentIds: z.ZodArray<z.ZodString>;
+        createdAt: z.ZodString;
+        proposer: z.ZodString;
+        baseSourceHash: z.ZodString;
+        candidateHash: z.ZodString;
+        components: z.ZodRecord<z.ZodString, z.ZodString>;
+        sourceBytes: z.ZodNumber;
+        rationale: z.ZodExactOptional<z.ZodString>;
+        proposalUsage: z.ZodExactOptional<z.ZodObject<{
+            input: z.ZodNumber;
+            output: z.ZodNumber;
+            cacheRead: z.ZodNumber;
+            cacheCreation: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    manifest: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        experimentId: z.ZodString;
+        campaignId: z.ZodString;
+        campaignPath: z.ZodString;
+        campaignHash: z.ZodString;
+        campaignSnapshot: z.ZodObject<{
+            id: z.ZodString;
+            subject: z.ZodObject<{
+                root: z.ZodString;
+                mutable: z.ZodArray<z.ZodString>;
+                exclude: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                validate: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    command: z.ZodString;
+                    args: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    timeoutMs: z.ZodDefault<z.ZodNumber>;
+                    unsafeHostExecution: z.ZodLiteral<true>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>;
+            evaluation: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"agent">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"grader">;
+                annotations: z.ZodString;
+                allowedAssertionAdditions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+                scoreTolerance: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>], "kind">>;
+            profiles: z.ZodArray<z.ZodString>;
+            splits: z.ZodObject<{
+                train: z.ZodOptional<z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>>;
+                validation: z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>;
+                test: z.ZodOptional<z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>>;
+            }, z.core.$strict>;
+            repetitions: z.ZodDefault<z.ZodObject<{
+                search: z.ZodDefault<z.ZodNumber>;
+                promotion: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            variants: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodOptional<z.ZodString>;
+                parentIds: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                components: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
+            }, z.core.$strict>>>;
+            objectives: z.ZodDefault<z.ZodObject<{
+                constraints: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    metric: z.ZodString;
+                    min: z.ZodOptional<z.ZodNumber>;
+                    max: z.ZodOptional<z.ZodNumber>;
+                    profile: z.ZodOptional<z.ZodString>;
+                    split: z.ZodDefault<z.ZodEnum<{
+                        test: "test";
+                        train: "train";
+                        validation: "validation";
+                    }>>;
+                    confidence: z.ZodOptional<z.ZodObject<{
+                        method: z.ZodEnum<{
+                            hoeffding: "hoeffding";
+                            "percentile-bootstrap": "percentile-bootstrap";
+                            wilson: "wilson";
+                        }>;
+                        level: z.ZodDefault<z.ZodNumber>;
+                        resamples: z.ZodDefault<z.ZodNumber>;
+                        seed: z.ZodDefault<z.ZodNumber>;
+                        minSamples: z.ZodDefault<z.ZodNumber>;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>>;
+                minimize: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    metric: z.ZodString;
+                    profile: z.ZodOptional<z.ZodString>;
+                    split: z.ZodDefault<z.ZodEnum<{
+                        test: "test";
+                        train: "train";
+                        validation: "validation";
+                    }>>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>>;
+            optimizer: z.ZodOptional<z.ZodObject<{
+                algorithm: z.ZodLiteral<"reflective">;
+                config: z.ZodOptional<z.ZodUnknown>;
+                candidateSelection: z.ZodDefault<z.ZodEnum<{
+                    "current-best": "current-best";
+                    pareto: "pareto";
+                }>>;
+                mergeFrontier: z.ZodDefault<z.ZodBoolean>;
+                promotionTopK: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            budget: z.ZodDefault<z.ZodObject<{
+                maxCandidates: z.ZodDefault<z.ZodNumber>;
+                patience: z.ZodDefault<z.ZodNumber>;
+                maxMetricCalls: z.ZodDefault<z.ZodNumber>;
+                maxJudgeCalls: z.ZodDefault<z.ZodNumber>;
+                maxProposalCalls: z.ZodDefault<z.ZodNumber>;
+                maxTokens: z.ZodDefault<z.ZodNumber>;
+                maxCostUsd: z.ZodDefault<z.ZodNumber>;
+                maxUnpricedModelCalls: z.ZodDefault<z.ZodNumber>;
+                maxWallTimeMs: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>;
+        resolvedProfiles: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        evaluatorVersion: z.ZodLiteral<1>;
+        luxVersion: z.ZodString;
+        evaluationKind: z.ZodDefault<z.ZodEnum<{
+            agent: "agent";
+            grader: "grader";
+        }>>;
+        evaluationHash: z.ZodExactOptional<z.ZodString>;
+        mode: z.ZodEnum<{
+            experiment: "experiment";
+            optimize: "optimize";
+        }>;
+        status: z.ZodEnum<{
+            aborted: "aborted";
+            failed: "failed";
+            passed: "passed";
+            running: "running";
+        }>;
+        startedAt: z.ZodString;
+        endedAt: z.ZodNullable<z.ZodString>;
+        activeTimeMs: z.ZodDefault<z.ZodNumber>;
+        activeSegmentStartedAt: z.ZodExactOptional<z.ZodString>;
+        subjectRoot: z.ZodString;
+        baseSourceHash: z.ZodString;
+        candidateIntegrityVersion: z.ZodExactOptional<z.ZodLiteral<1>>;
+        candidates: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            path: z.ZodString;
+            candidateHash: z.ZodString;
+            sourceBytes: z.ZodNumber;
+            candidateEnvelopeHash: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>;
+        evaluations: z.ZodArray<z.ZodObject<{
+            profileId: z.ZodString;
+            split: z.ZodEnum<{
+                test: "test";
+                train: "train";
+                validation: "validation";
+            }>;
+            suiteDir: z.ZodExactOptional<z.ZodString>;
+            passRate: z.ZodNumber;
+            casePassRates: z.ZodRecord<z.ZodString, z.ZodNumber>;
+            metrics: z.ZodRecord<z.ZodString, z.ZodNumber>;
+            metricCalls: z.ZodExactOptional<z.ZodNumber>;
+            judgeCalls: z.ZodExactOptional<z.ZodNumber>;
+            unpricedModelCalls: z.ZodExactOptional<z.ZodNumber>;
+            usage: z.ZodExactOptional<z.ZodObject<{
+                input: z.ZodNumber;
+                output: z.ZodNumber;
+                cacheRead: z.ZodNumber;
+                cacheCreation: z.ZodNumber;
+            }, z.core.$strip>>;
+            costUsd: z.ZodExactOptional<z.ZodNumber>;
+            samples: z.ZodExactOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber>>>;
+            failedCases: z.ZodArray<z.ZodObject<{
+                caseId: z.ZodString;
+                score: z.ZodNullable<z.ZodNumber>;
+                feedback: z.ZodArray<z.ZodString>;
+            }, z.core.$strip>>;
+            candidateId: z.ZodString;
+            repetitions: z.ZodNumber;
+            summaryPath: z.ZodExactOptional<z.ZodString>;
+            evidencePath: z.ZodExactOptional<z.ZodString>;
+        }, z.core.$strip>>;
+        accountedEvaluationUnits: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        accountingJournalEntries: z.ZodDefault<z.ZodNumber>;
+        budgetUsage: z.ZodObject<{
+            metricCalls: z.ZodNumber;
+            judgeCalls: z.ZodNumber;
+            unpricedModelCalls: z.ZodOptional<z.ZodNumber>;
+            proposalCalls: z.ZodNumber;
+            usage: z.ZodObject<{
+                input: z.ZodNumber;
+                output: z.ZodNumber;
+                cacheRead: z.ZodNumber;
+                cacheCreation: z.ZodNumber;
+            }, z.core.$strip>;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>;
+        championCandidateId: z.ZodExactOptional<z.ZodString>;
+        selection: z.ZodExactOptional<z.ZodObject<{
+            decidedAt: z.ZodString;
+            decisionCandidateIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            promotionFinalistCandidateIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            rankedCandidateIds: z.ZodArray<z.ZodString>;
+            eligibleCandidateIds: z.ZodArray<z.ZodString>;
+            championCandidateId: z.ZodExactOptional<z.ZodString>;
+            constraints: z.ZodArray<z.ZodObject<{
+                candidateId: z.ZodString;
+                metric: z.ZodString;
+                profile: z.ZodExactOptional<z.ZodString>;
+                split: z.ZodEnum<{
+                    test: "test";
+                    train: "train";
+                    validation: "validation";
+                }>;
+                values: z.ZodArray<z.ZodNumber>;
+                passed: z.ZodBoolean;
+                interval: z.ZodExactOptional<z.ZodObject<{
+                    method: z.ZodEnum<{
+                        hoeffding: "hoeffding";
+                        "percentile-bootstrap": "percentile-bootstrap";
+                        wilson: "wilson";
+                    }>;
+                    level: z.ZodNumber;
+                    estimate: z.ZodNumber;
+                    lower: z.ZodNumber;
+                    upper: z.ZodNumber;
+                    n: z.ZodNumber;
+                    resamples: z.ZodNumber;
+                }, z.core.$strict>>;
+                note: z.ZodExactOptional<z.ZodString>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        errorMessage: z.ZodExactOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strict>;
+export type CampaignExecutionResult = z.infer<typeof CampaignExecutionResultSchema>;
+export declare const ExperimentCampaignPlanSchema: z.ZodObject<{
+    campaignId: z.ZodString;
+    mode: z.ZodEnum<{
+        experiment: "experiment";
+        optimize: "optimize";
+    }>;
+    evaluationKind: z.ZodEnum<{
+        agent: "agent";
+        grader: "grader";
+    }>;
+    subjectRoot: z.ZodString;
+    mutableComponents: z.ZodArray<z.ZodString>;
+    profiles: z.ZodArray<z.ZodString>;
+    splits: z.ZodArray<z.ZodObject<{
+        split: z.ZodEnum<{
+            test: "test";
+            train: "train";
+            validation: "validation";
+        }>;
+        examples: z.ZodNumber;
+    }, z.core.$strict>>;
+    projected: z.ZodObject<{
+        candidates: z.ZodNumber;
+        metricCalls: z.ZodNumber;
+        judgeCalls: z.ZodNumber;
+        proposalCalls: z.ZodNumber;
+    }, z.core.$strict>;
+    budget: z.ZodObject<{
+        maxCandidates: z.ZodDefault<z.ZodNumber>;
+        patience: z.ZodDefault<z.ZodNumber>;
+        maxMetricCalls: z.ZodDefault<z.ZodNumber>;
+        maxJudgeCalls: z.ZodDefault<z.ZodNumber>;
+        maxProposalCalls: z.ZodDefault<z.ZodNumber>;
+        maxTokens: z.ZodDefault<z.ZodNumber>;
+        maxCostUsd: z.ZodDefault<z.ZodNumber>;
+        maxUnpricedModelCalls: z.ZodDefault<z.ZodNumber>;
+        maxWallTimeMs: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strict>;
+    fitsProjectedCallBudgets: z.ZodBoolean;
+    warnings: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export type ExperimentCampaignPlan = z.infer<typeof ExperimentCampaignPlanSchema>;
+//# sourceMappingURL=experiment-contracts.d.ts.map

package/lib/orchestrator/experiment-contracts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-contracts.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA4B9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}

package/lib/orchestrator/experiment-contracts.js

@@ -0,0 +1,38 @@
+import { z } from "zod";
+import { CandidateSchema, ExperimentBudgetSchema } from "../core/index.js";
+import { ExperimentRunSchema } from "./experiment-runs.js";
+export const CampaignExecutionResultSchema = z
+    .object({
+    experimentDir: z.string(),
+    champion: CandidateSchema.nullable(),
+    manifest: ExperimentRunSchema,
+})
+    .strict();
+export const ExperimentCampaignPlanSchema = z
+    .object({
+    campaignId: z.string(),
+    mode: z.enum(["experiment", "optimize"]),
+    evaluationKind: z.enum(["agent", "grader"]),
+    subjectRoot: z.string(),
+    mutableComponents: z.array(z.string()),
+    profiles: z.array(z.string()),
+    splits: z.array(z
+        .object({
+        split: z.enum(["train", "validation", "test"]),
+        examples: z.number().int().nonnegative(),
+    })
+        .strict()),
+    projected: z
+        .object({
+        candidates: z.number().int().nonnegative(),
+        metricCalls: z.number().int().nonnegative(),
+        judgeCalls: z.number().int().nonnegative(),
+        proposalCalls: z.number().int().nonnegative(),
+    })
+        .strict(),
+    budget: ExperimentBudgetSchema,
+    fitsProjectedCallBudgets: z.boolean(),
+    warnings: z.array(z.string()),
+})
+    .strict();
+//# sourceMappingURL=experiment-contracts.js.map

package/lib/orchestrator/experiment-contracts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-contracts.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC;KAC3C,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IACzB,QAAQ,EAAE,eAAe,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,mBAAmB;CAC9B,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC;KAC1C,MAAM,CAAC;IACN,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IACxC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7B,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC;SACE,MAAM,CAAC;QACN,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAC9C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;SACD,MAAM,EAAE,CACZ;IACD,SAAS,EAAE,CAAC;SACT,MAAM,CAAC;QACN,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC1C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC3C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC1C,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;KAC9C,CAAC;SACD,MAAM,EAAE;IACX,MAAM,EAAE,sBAAsB;IAC9B,wBAAwB,EAAE,CAAC,CAAC,OAAO,EAAE;IACrC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC9B,CAAC;KACD,MAAM,EAAE,CAAC"}

package/lib/orchestrator/experiment-corpus.d.ts

@@ -0,0 +1,21 @@
+import { type Candidate, type ExperimentCampaign, type ExperimentSelector } from "../core/index.js";
+import { discoverCases } from "./discover.js";
+import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import { loadEvalCase } from "./loader.js";
+type ExperimentSplit = "train" | "validation" | "test";
+export declare const selectorsBySplit: (campaign: ExperimentCampaign) => Array<{
+    name: ExperimentSplit;
+    selector: ExperimentSelector;
+}>;
+export declare const caseRubricCliReferences: (execution: CampaignExecutionOptions, authored: Awaited<ReturnType<typeof loadEvalCase>>) => Array<{
+    assertionId: string;
+    profileId: string;
+    provider: "claude-code" | "codex";
+    command: string;
+}>;
+export declare const agentCorpusHash: (execution: CampaignExecutionOptions, cases: Awaited<ReturnType<typeof discoverCases>>["cases"], splits: ReturnType<typeof selectorsBySplit>) => Promise<string>;
+export declare const graderCorpusHash: (execution: CampaignExecutionOptions, splits: ReturnType<typeof selectorsBySplit>) => Promise<string>;
+export declare const prepareCorpus: (execution: CampaignExecutionOptions) => Promise<EvaluationCorpus>;
+export declare const assertGraderSubject: (execution: CampaignExecutionOptions, baseline: Candidate) => Promise<void>;
+export {};
+//# sourceMappingURL=experiment-corpus.d.ts.map

package/lib/orchestrator/experiment-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-corpus.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,SAAS,EAEd,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAExB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAW1F,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAK3C,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAEvD,eAAO,MAAM,gBAAgB,aACjB,kBAAkB,KAC3B,KAAK,CAAC;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAA;CAAE,CAI/D,CAAC;AA0BF,eAAO,MAAM,uBAAuB,cACvB,wBAAwB,YACzB,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,KACjD,KAAK,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;CACjB,CAsBA,CAAC;AAoBF,eAAO,MAAM,eAAe,cACf,wBAAwB,SAC5B,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,UACjD,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CAsEhB,CAAC;AAEF,eAAO,MAAM,gBAAgB,cAChB,wBAAwB,UAC3B,UAAU,CAAC,OAAO,gBAAgB,CAAC,KAC1C,OAAO,CAAC,MAAM,CAgFhB,CAAC;AAEF,eAAO,MAAM,aAAa,cACb,wBAAwB,KAClC,OAAO,CAAC,gBAAgB,CA6E1B,CAAC;AAEF,eAAO,MAAM,mBAAmB,cACnB,wBAAwB,YACzB,SAAS,KAClB,OAAO,CAAC,IAAI,CAyDd,CAAC"}