@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment.js

@@ -0,0 +1,639 @@
+import { randomUUID } from "node:crypto";
+import { rm } from "node:fs/promises";
+import { bootstrapMeanConfidenceInterval, } from "../core/index.js";
+import { createReflectiveOptimizer } from "../optimization/reflective.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
+import { executeCampaignLifecycle } from "./campaign-lifecycle.js";
+import { captureSeedCandidate, createCandidate, hashSubjectTree, materializeCandidate, validateCandidate, } from "./candidates.js";
+import { selectCases } from "./discover.js";
+import { boundedExecutionSignal, ensureBudget, throwIfAborted } from "./experiment-budget.js";
+import { CampaignExecutionResultSchema, ExperimentCampaignPlanSchema, } from "./experiment-contracts.js";
+import { agentCorpusHash, assertGraderSubject, graderCorpusHash, prepareCorpus, selectorsBySplit, } from "./experiment-corpus.js";
+import { evaluateAgentCandidate, evaluateOfflineGraderCandidate, invokeWithVerifiedExecutionIdentity, } from "./experiment-evaluation.js";
+import { finishExperiment } from "./experiment-finish.js";
+import { evaluationIdentity, resolvedProfileSnapshot, runtimeCliReferences, } from "./experiment-identity.js";
+import { loadExperimentCandidates, loadExperimentRun, openExperimentRun, resumeExperimentRun, } from "./experiment-runs.js";
+import { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes, LUX_VERSION, runtimeCliIdentityHashForReferences, } from "./experiment-runtime-identity.js";
+import { candidateEvaluations, rankCandidates, searchFrontier, } from "./experiment-selection.js";
+import { loadEvalCase } from "./loader.js";
+import { executeOptimizationLifecycle } from "./optimization-lifecycle.js";
+export { CampaignExecutionResultSchema, ExperimentCampaignPlanSchema, } from "./experiment-contracts.js";
+export { deriveExperimentSelection, ExperimentDecisionCandidateSchema, paretoFrontier, rankCandidates, } from "./experiment-selection.js";
+const evaluateAcross = async (options) => {
+    const boundedSignal = boundedExecutionSignal(options.execution, options.experiment);
+    const boundedExecution = { ...options.execution, abortSignal: boundedSignal };
+    throwIfAborted(boundedSignal);
+    ensureBudget(options.execution.loaded.campaign, options.experiment);
+    const currentCorpusHash = options.corpus.kind === "agent"
+        ? await agentCorpusHash(options.execution, options.corpus.cases, selectorsBySplit(options.execution.loaded.campaign))
+        : await graderCorpusHash(options.execution, selectorsBySplit(options.execution.loaded.campaign));
+    if (currentCorpusHash !== options.corpus.hash) {
+        throw new Error("experiment evaluation corpus changed while the campaign was running");
+    }
+    const materializedRoot = await materializeCandidate(options.execution.loaded.subjectRoot, options.execution.loaded.campaign.subject, options.candidate);
+    try {
+        await validateCandidate(materializedRoot, options.execution.loaded.campaign.subject, boundedSignal, options.verifyExecutionIdentity);
+        throwIfAborted(boundedSignal);
+        const evaluations = [];
+        for (const profileId of options.execution.loaded.campaign.profiles) {
+            for (const split of options.splits) {
+                throwIfAborted(boundedSignal);
+                evaluations.push(options.corpus.kind === "agent"
+                    ? await evaluateAgentCandidate({
+                        execution: boundedExecution,
+                        experiment: options.experiment,
+                        candidate: options.candidate,
+                        allCases: options.corpus.cases,
+                        profileId,
+                        verifyExecutionIdentity: options.verifyExecutionIdentity,
+                        ...split,
+                    })
+                    : await evaluateOfflineGraderCandidate({
+                        execution: boundedExecution,
+                        experiment: options.experiment,
+                        candidate: options.candidate,
+                        materializedRoot,
+                        profileId,
+                        verifyExecutionIdentity: options.verifyExecutionIdentity,
+                        ...split,
+                    }));
+                throwIfAborted(boundedSignal);
+                ensureBudget(options.execution.loaded.campaign, options.experiment);
+            }
+        }
+        const finalCorpusHash = options.corpus.kind === "agent"
+            ? await agentCorpusHash(options.execution, options.corpus.cases, selectorsBySplit(options.execution.loaded.campaign))
+            : await graderCorpusHash(options.execution, selectorsBySplit(options.execution.loaded.campaign));
+        if (finalCorpusHash !== options.corpus.hash) {
+            throw new Error("experiment evaluation corpus changed while candidate evaluation was running");
+        }
+        const finalSourceHash = await hashSubjectTree(options.execution.loaded.subjectRoot, options.execution.loaded.campaign.subject);
+        if (finalSourceHash !== options.candidate.baseSourceHash) {
+            throw new Error("experiment subject source changed while candidate evaluation was running");
+        }
+        return evaluations;
+    }
+    finally {
+        await rm(materializedRoot, { recursive: true, force: true });
+    }
+};
+export const planExperimentCampaign = async (execution, mode) => {
+    const campaign = execution.loaded.campaign;
+    if (mode === "optimize") {
+        if (!campaign.optimizer) {
+            throw new Error(`experiment '${campaign.id}' has no optimizer configuration`);
+        }
+        createReflectiveOptimizer(campaign.optimizer.config);
+    }
+    const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
+    await assertGraderSubject(execution, baseline);
+    const corpus = await prepareCorpus(execution);
+    // Plan is the deterministic preflight for execution. Build the exact same
+    // identity now so profile, behavior-tree, and custom CLI failures happen
+    // before a user approves any model spend.
+    await evaluationIdentity(execution, corpus, mode);
+    const splitSelectors = selectorsBySplit(campaign);
+    const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
+    const splits = splitSelectors.map(({ name, selector }) => ({
+        split: name,
+        examples: corpus.kind === "agent"
+            ? selectCases(corpus.cases, selector).length
+            : selectGraderAnnotations(graderAnnotations, selector).length,
+    }));
+    const profiles = campaign.profiles.length;
+    const count = (names) => splits
+        .filter((split) => names.includes(split.split))
+        .reduce((total, split) => total + split.examples, 0);
+    const projectedCandidates = mode === "experiment" ? 1 + campaign.variants.length : campaign.budget.maxCandidates;
+    const projectedMetricCalls = mode === "experiment"
+        ? projectedCandidates *
+            profiles *
+            count(["train", "validation", "test"]) *
+            campaign.repetitions.promotion
+        : projectedCandidates *
+            profiles *
+            count(["train", "validation"]) *
+            campaign.repetitions.search +
+            (1 + (campaign.optimizer?.promotionTopK ?? 1)) *
+                profiles *
+                count(["validation", "test"]) *
+                campaign.repetitions.promotion;
+    let maxRubricAssertions = 0;
+    if (corpus.kind === "grader") {
+        // Grader candidates may grow any rubric to the loader's hard cap.
+        maxRubricAssertions = 20;
+    }
+    else {
+        const selected = splitSelectors.flatMap((split) => selectCases(corpus.cases, split.selector));
+        maxRubricAssertions = Math.max(0, ...(await Promise.all(selected.map((evalCase) => loadEvalCase(evalCase.path)))).map((evalCase) => (evalCase.assertions ?? []).filter((assertion) => assertion.type === "rubric").length));
+    }
+    const projectedJudgeCalls = projectedMetricCalls * maxRubricAssertions * 2;
+    const projectedProposalCalls = mode === "optimize" ? campaign.budget.maxProposalCalls : 0;
+    const warnings = campaign.objectives.constraints.flatMap((constraint) => {
+        if (!constraint.confidence)
+            return [];
+        const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
+        let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
+        if (corpus.kind === "grader" && splitSelector) {
+            const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+            if (constraint.metric === "grader.macroFalsePositiveRate") {
+                examples = selected.filter((annotation) => [
+                    annotation.expected.casePassed,
+                    ...Object.values(annotation.expected.assertions).map((label) => label.passed),
+                ].includes(false)).length;
+            }
+            else if (constraint.metric === "grader.macroFalseNegativeRate") {
+                examples = selected.filter((annotation) => [
+                    annotation.expected.casePassed,
+                    ...Object.values(annotation.expected.assertions).map((label) => label.passed),
+                ].includes(true)).length;
+            }
+            else if (constraint.metric === "grader.scoreMae") {
+                examples = selected.filter((annotation) => annotation.expected.caseScore !== undefined ||
+                    Object.values(annotation.expected.assertions).some((label) => label.score !== undefined)).length;
+            }
+            else if (constraint.metric === "grader.caseAgreement") {
+                examples = selected.filter((annotation) => annotation.expected.casePassed !== undefined ||
+                    annotation.expected.caseScore !== undefined).length;
+            }
+            else if (constraint.metric === "grader.assertionAgreement") {
+                examples = selected.filter((annotation) => Object.keys(annotation.expected.assertions).length > 0).length;
+            }
+            else if (constraint.metric === "grader.abstentionRate" ||
+                constraint.metric === "grader.coverage") {
+                examples = selected.filter((annotation) => annotation.expected.casePassed !== undefined ||
+                    Object.keys(annotation.expected.assertions).length > 0).length;
+            }
+        }
+        if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
+            examples = 1;
+        }
+        const notes = [];
+        if (examples < constraint.confidence.minSamples) {
+            notes.push(`${constraint.split} metric '${constraint.metric}' has ${examples} independent samples, below confidence minSamples=${constraint.confidence.minSamples}`);
+        }
+        else if (constraint.confidence.method === "wilson" ||
+            constraint.confidence.method === "hoeffding") {
+            const bestForMin = bootstrapMeanConfidenceInterval(Array(examples).fill(1), constraint.confidence);
+            const bestForMax = bootstrapMeanConfidenceInterval(Array(examples).fill(0), constraint.confidence);
+            if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
+                notes.push(`${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`);
+            }
+            if (constraint.max !== undefined && bestForMax.upper > constraint.max) {
+                notes.push(`${constraint.split} metric '${constraint.metric}' cannot attain confidence upper bound ${constraint.max} with only ${examples} samples (best ${bestForMax.upper.toFixed(3)})`);
+            }
+        }
+        return notes;
+    });
+    warnings.push("token, reported-cost, and wall-time limits are enforced from observed runtime usage and cannot be proven by the no-call plan");
+    return ExperimentCampaignPlanSchema.parse({
+        campaignId: campaign.id,
+        mode,
+        evaluationKind: campaign.evaluation.kind,
+        subjectRoot: execution.loaded.subjectRoot,
+        mutableComponents: Object.keys(baseline.components).sort(),
+        profiles: [...campaign.profiles],
+        splits,
+        projected: {
+            candidates: projectedCandidates,
+            metricCalls: projectedMetricCalls,
+            judgeCalls: projectedJudgeCalls,
+            proposalCalls: projectedProposalCalls,
+        },
+        budget: campaign.budget,
+        fitsProjectedCallBudgets: projectedCandidates <= campaign.budget.maxCandidates &&
+            projectedMetricCalls <= campaign.budget.maxMetricCalls &&
+            projectedJudgeCalls <= campaign.budget.maxJudgeCalls &&
+            projectedProposalCalls <= campaign.budget.maxProposalCalls,
+        warnings,
+    });
+};
+const prepareExecution = async (execution, mode) => {
+    const campaign = execution.loaded.campaign;
+    const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
+    await assertGraderSubject(execution, baseline);
+    const corpus = await prepareCorpus(execution);
+    const runtimeReferences = await runtimeCliReferences(execution, corpus, mode);
+    const expectedRuntimeCliIdentityHash = await runtimeCliIdentityHashForReferences(runtimeReferences);
+    const currentEvaluationHash = await evaluationIdentity(execution, corpus, mode, expectedRuntimeCliIdentityHash);
+    const verifyExecutionIdentity = async () => {
+        const observed = await runtimeCliIdentityHashForReferences(runtimeReferences);
+        if (observed !== expectedRuntimeCliIdentityHash) {
+            throw new Error("experiment CLI identity changed during execution");
+        }
+    };
+    await verifyExecutionIdentity();
+    if (execution.resumeFrom) {
+        const saved = await loadExperimentRun(execution.resumeFrom);
+        if (!saved) {
+            throw new Error(`experiment manifest not found or invalid at ${execution.resumeFrom}`);
+        }
+        if (saved.mode !== mode) {
+            throw new Error(`cannot resume ${saved.mode} run with lux ${mode}`);
+        }
+        if (saved.campaignId !== campaign.id || saved.campaignHash !== execution.loaded.campaignHash) {
+            throw new Error("cannot resume: the experiment campaign definition changed");
+        }
+        if (saved.evaluationKind !== campaign.evaluation.kind) {
+            throw new Error("cannot resume: the experiment evaluation kind changed");
+        }
+        if (saved.evaluationHash !== currentEvaluationHash) {
+            throw new Error("cannot resume: the evaluation corpus or profile configuration changed");
+        }
+        if (saved.subjectRoot !== execution.loaded.subjectRoot ||
+            saved.baseSourceHash !== baseline.baseSourceHash) {
+            throw new Error("cannot resume: the experiment subject source changed");
+        }
+        const experiment = await resumeExperimentRun(execution.resumeFrom);
+        try {
+            const candidates = await loadExperimentCandidates(experiment.path);
+            const savedBaseline = candidates.find((candidate) => candidate.id === "baseline");
+            if (!savedBaseline)
+                throw new Error("cannot resume: baseline candidate is missing");
+            return {
+                baseline: savedBaseline,
+                candidates,
+                experiment,
+                corpus,
+                verifyExecutionIdentity,
+            };
+        }
+        catch (error) {
+            await experiment.release();
+            throw error;
+        }
+    }
+    const experiment = await openExperimentRun({
+        runsRoot: execution.runsRoot ?? execution.config.runsRoot,
+        campaignId: campaign.id,
+        campaignPath: execution.loaded.path,
+        campaignHash: execution.loaded.campaignHash,
+        campaignSnapshot: campaign,
+        resolvedProfiles: await resolvedProfileSnapshot(execution),
+        luxVersion: LUX_VERSION,
+        subjectRoot: execution.loaded.subjectRoot,
+        baseSourceHash: baseline.baseSourceHash,
+        mode,
+        evaluationKind: campaign.evaluation.kind,
+        evaluationHash: currentEvaluationHash,
+    });
+    try {
+        await experiment.saveCandidate(baseline);
+        return {
+            baseline,
+            candidates: [baseline],
+            experiment,
+            corpus,
+            verifyExecutionIdentity,
+        };
+    }
+    catch (error) {
+        await experiment.release();
+        throw error;
+    }
+};
+const splitPlan = (campaign, repetitions, includeTrain) => [
+    ...(includeTrain && campaign.splits.train
+        ? [{ split: "train", selector: campaign.splits.train, repetitions }]
+        : []),
+    { split: "validation", selector: campaign.splits.validation, repetitions },
+    ...(campaign.splits.test
+        ? [{ split: "test", selector: campaign.splits.test, repetitions }]
+        : []),
+];
+class CampaignExecution {
+    prepared;
+    execution;
+    mode;
+    worker;
+    constructor(execution, mode, worker) {
+        this.execution = execution;
+        this.mode = mode;
+        this.worker = worker;
+    }
+    async prepare() {
+        this.prepared = await prepareExecution(this.execution, this.mode);
+    }
+    execute() {
+        return this.worker(this.execution, this.requirePrepared());
+    }
+    async complete(work) {
+        const prepared = this.requirePrepared();
+        const champion = await finishExperiment(prepared.experiment, this.execution.loaded.campaign, work.candidates, work.selectionOptions);
+        return CampaignExecutionResultSchema.parse({
+            experimentDir: prepared.experiment.path,
+            champion,
+            manifest: prepared.experiment.manifest,
+        });
+    }
+    resultStatus(result) {
+        return result.manifest.status === "passed" ? "passed" : "failed";
+    }
+    failureStatus(_error) {
+        return this.execution.abortSignal?.aborted ? "aborted" : "failed";
+    }
+    async recordFailure(status, error) {
+        if (!this.prepared)
+            return;
+        const { manifest } = this.prepared.experiment;
+        manifest.status = status;
+        manifest.endedAt = new Date().toISOString();
+        manifest.errorMessage = error instanceof Error ? error.message : String(error);
+        await this.prepared.experiment.write();
+    }
+    async close(_primaryError) {
+        await this.prepared?.experiment.release();
+    }
+    requirePrepared() {
+        if (!this.prepared)
+            throw new Error("campaign lifecycle entered execution before preparation");
+        return this.prepared;
+    }
+}
+const runStaticCampaignWork = async (execution, prepared) => {
+    const campaign = execution.loaded.campaign;
+    const { baseline, candidates: savedCandidates, experiment, corpus, verifyExecutionIdentity, } = prepared;
+    const candidates = [
+        baseline,
+        ...campaign.variants.map((variant) => {
+            const saved = savedCandidates.find((savedCandidate) => savedCandidate.id === variant.id);
+            if (saved)
+                return saved;
+            const candidate = createCandidate(baseline, { updates: variant.components }, campaign.subject, { id: variant.id, proposer: "manual" });
+            candidate.parentIds = variant.parentIds.length > 0 ? variant.parentIds : [baseline.id];
+            return candidate;
+        }),
+    ];
+    for (const candidate of candidates) {
+        throwIfAborted(execution.abortSignal);
+        await experiment.saveCandidate(candidate);
+        await evaluateAcross({
+            execution,
+            experiment,
+            candidate,
+            corpus,
+            verifyExecutionIdentity,
+            splits: splitPlan(campaign, campaign.repetitions.promotion, true),
+        });
+    }
+    return { candidates };
+};
+const createOptimizationContext = (execution, prepared) => {
+    const campaign = execution.loaded.campaign;
+    const configuration = campaign.optimizer;
+    if (!configuration) {
+        throw new Error(`experiment '${campaign.id}' has no optimizer configuration`);
+    }
+    return {
+        execution,
+        prepared,
+        configuration,
+        optimizer: createReflectiveOptimizer(configuration.config),
+        searchSplits: splitPlan(campaign, campaign.repetitions.search, true).filter((entry) => entry.split !== "test"),
+    };
+};
+const rankedSearchCandidates = (context, candidates) => rankCandidates(context.execution.loaded.campaign, candidates, context.prepared.experiment.manifest, ["train", "validation"]);
+const optimizationFrontier = (context, candidates) => {
+    if (context.configuration.candidateSelection === "pareto") {
+        return searchFrontier(context.execution.loaded.campaign, candidates, context.prepared.experiment.manifest);
+    }
+    return [rankedSearchCandidates(context, candidates)[0] ?? context.prepared.baseline];
+};
+const alternativeCandidates = (context, frontier, current) => context.configuration.mergeFrontier
+    ? frontier.filter((candidate) => candidate.id !== current.id).slice(0, 2)
+    : [];
+const trainEvaluations = (experiment, candidateId) => candidateEvaluations(experiment.manifest, candidateId).filter((evaluation) => evaluation.split === "train");
+const requestOptimizationProposals = async (context, state, current, alternatives) => {
+    const { execution, prepared, optimizer } = context;
+    const { experiment, verifyExecutionIdentity, candidates } = prepared;
+    const campaign = execution.loaded.campaign;
+    const proposalSignal = boundedExecutionSignal(execution, experiment);
+    const proposalAttemptIds = [];
+    let pendingAccounting;
+    const input = {
+        campaign,
+        candidate: current,
+        observations: trainEvaluations(experiment, current.id),
+        remainingCandidates: campaign.budget.maxCandidates - candidates.length,
+        cwd: execution.loaded.directory,
+        abortSignal: proposalSignal,
+        recordModelAttemptStart: async (kind) => {
+            await verifyExecutionIdentity();
+            proposalAttemptIds.push(await experiment.startModelAttempt(kind));
+        },
+        completeModelAttempts: async () => {
+            await verifyExecutionIdentity();
+            if (pendingAccounting) {
+                await experiment.settleModelAttempts(`proposal-spend-${randomUUID()}`, pendingAccounting, proposalAttemptIds);
+                pendingAccounting = undefined;
+            }
+            else {
+                await experiment.completeModelAttempts(proposalAttemptIds);
+            }
+            proposalAttemptIds.length = 0;
+            ensureBudget(campaign, experiment);
+        },
+        recordModelCall: async (accounting) => {
+            await verifyExecutionIdentity();
+            pendingAccounting = {
+                usage: accounting.usage,
+                costUsd: accounting.costUsd ?? 0,
+                unpricedModelCalls: accounting.costUsd === undefined ? 1 : 0,
+            };
+        },
+    };
+    if (alternatives.length > 0) {
+        input.alternatives = alternatives;
+        input.alternativeObservations = Object.fromEntries(alternatives.map((candidate) => [candidate.id, trainEvaluations(experiment, candidate.id)]));
+    }
+    const proposals = await invokeWithVerifiedExecutionIdentity(verifyExecutionIdentity, async () => {
+        ensureBudget(campaign, experiment, { proposalCalls: 1 });
+        await experiment.recordAccounting(`proposal:${state.iteration}:${randomUUID()}`, {
+            proposalCalls: 1,
+        });
+        await experiment.write();
+        return optimizer.propose(input);
+    });
+    throwIfAborted(proposalSignal);
+    ensureBudget(campaign, experiment);
+    return proposals;
+};
+const addOptimizationCandidates = async (context, current, alternatives, proposals) => {
+    const { execution, prepared, configuration, searchSplits } = context;
+    const { candidates, experiment, corpus, verifyExecutionIdentity } = prepared;
+    const campaign = execution.loaded.campaign;
+    let added = 0;
+    for (const proposal of proposals) {
+        const candidate = createCandidate(current, proposal, campaign.subject, {
+            proposer: `reflective:${configuration.algorithm}`,
+        });
+        if (alternatives.length > 0) {
+            candidate.parentIds = [current.id, ...alternatives.map((entry) => entry.id)];
+        }
+        if (candidates.some((entry) => entry.candidateHash === candidate.candidateHash))
+            continue;
+        candidates.push(candidate);
+        added++;
+        await experiment.saveCandidate(candidate);
+        await evaluateAcross({
+            execution,
+            experiment,
+            candidate,
+            corpus,
+            verifyExecutionIdentity,
+            splits: searchSplits,
+        });
+        if (candidates.length >= campaign.budget.maxCandidates)
+            break;
+    }
+    return added;
+};
+const advanceSearchState = (context, state, added, previousFrontier) => {
+    const { candidates, baseline } = context.prepared;
+    const nextBest = rankedSearchCandidates(context, candidates)[0]?.id ?? baseline.id;
+    const expandedFrontier = context.configuration.candidateSelection === "pareto" &&
+        optimizationFrontier(context, candidates).some((candidate) => !previousFrontier.has(candidate.candidateHash));
+    return {
+        iteration: state.iteration + 1,
+        staleIterations: added > 0 && (nextBest !== state.bestId || expandedFrontier) ? 0 : state.staleIterations + 1,
+        bestId: nextBest,
+    };
+};
+const planOptimizationIteration = (context, state) => {
+    throwIfAborted(context.execution.abortSignal);
+    const frontier = optimizationFrontier(context, context.prepared.candidates);
+    const current = frontier[state.iteration % frontier.length] ?? context.prepared.baseline;
+    return {
+        current,
+        alternatives: alternativeCandidates(context, frontier, current),
+        frontierHashes: new Set(frontier.map((candidate) => candidate.candidateHash)),
+    };
+};
+const searchCanContinue = (context, state) => {
+    const { budget } = context.execution.loaded.campaign;
+    return (context.prepared.candidates.length < budget.maxCandidates &&
+        state.staleIterations < budget.patience &&
+        context.prepared.experiment.manifest.budgetUsage.proposalCalls < budget.maxProposalCalls &&
+        !context.execution.abortSignal?.aborted);
+};
+const evaluateSearchSeeds = async (context) => {
+    const { prepared, searchSplits } = context;
+    for (const candidate of prepared.candidates) {
+        await evaluateAcross({
+            execution: context.execution,
+            experiment: prepared.experiment,
+            candidate,
+            corpus: prepared.corpus,
+            verifyExecutionIdentity: prepared.verifyExecutionIdentity,
+            splits: searchSplits,
+        });
+    }
+    throwIfAborted(context.execution.abortSignal);
+};
+const initialOptimizationSearch = (context) => ({
+    iteration: 0,
+    staleIterations: 0,
+    bestId: rankedSearchCandidates(context, context.prepared.candidates)[0]?.id ??
+        context.prepared.baseline.id,
+});
+const evaluateOptimizationProposals = async (context, state, plan, proposals) => {
+    const added = await addOptimizationCandidates(context, plan.current, plan.alternatives, proposals);
+    return advanceSearchState(context, state, added, plan.frontierHashes);
+};
+const selectPromotionCandidates = (context) => {
+    const { prepared, configuration } = context;
+    const { baseline, candidates } = prepared;
+    const searchLeaders = rankedSearchCandidates(context, candidates)
+        .filter((candidate) => candidate.id !== baseline.id)
+        .slice(0, configuration.promotionTopK);
+    return { finalists: [baseline, ...searchLeaders] };
+};
+const evaluatePromotionCandidates = async (context, plan) => {
+    const { execution, prepared } = context;
+    const { baseline, experiment, corpus, verifyExecutionIdentity } = prepared;
+    const campaign = execution.loaded.campaign;
+    for (const candidate of plan.finalists) {
+        await evaluateAcross({
+            execution,
+            experiment,
+            candidate,
+            corpus,
+            verifyExecutionIdentity,
+            splits: [
+                {
+                    split: "validation",
+                    selector: campaign.splits.validation,
+                    repetitions: campaign.repetitions.promotion,
+                },
+            ],
+        });
+    }
+    const promoted = rankedSearchCandidates(context, plan.finalists)[0] ?? baseline;
+    return {
+        deployable: promoted.id === baseline.id ? [baseline] : [baseline, promoted],
+        finalistIds: plan.finalists.map((candidate) => candidate.id),
+    };
+};
+const evaluateDeploymentCandidates = async (context, plan) => {
+    const { execution, prepared } = context;
+    const { experiment, corpus, verifyExecutionIdentity } = prepared;
+    const campaign = execution.loaded.campaign;
+    if (campaign.splits.test) {
+        for (const candidate of plan.deployable) {
+            await evaluateAcross({
+                execution,
+                experiment,
+                candidate,
+                corpus,
+                verifyExecutionIdentity,
+                splits: [
+                    {
+                        split: "test",
+                        selector: campaign.splits.test,
+                        repetitions: campaign.repetitions.promotion,
+                    },
+                ],
+            });
+        }
+    }
+    return {
+        candidates: [...plan.deployable],
+        selectionOptions: {
+            promotionFinalistCandidateIds: [...plan.finalistIds],
+        },
+    };
+};
+const runOptimizationCampaignWork = async (execution, prepared) => {
+    const context = createOptimizationContext(execution, prepared);
+    return executeOptimizationLifecycle({
+        initialSearch: () => initialOptimizationSearch(context),
+        evaluateSearchSeeds: () => evaluateSearchSeeds(context),
+        searchCanContinue: (search) => searchCanContinue(context, search),
+        runSearchIteration: async (search) => {
+            const plan = planOptimizationIteration(context, search);
+            const proposals = await requestOptimizationProposals(context, search, plan.current, [
+                ...plan.alternatives,
+            ]);
+            return evaluateOptimizationProposals(context, search, plan, proposals);
+        },
+        promote: async () => evaluatePromotionCandidates(context, selectPromotionCandidates(context)),
+        test: (deployment) => evaluateDeploymentCandidates(context, deployment),
+    });
+};
+export const runExperimentCampaign = async (execution) => {
+    const campaign = execution.loaded.campaign;
+    const candidateCount = 1 + campaign.variants.length;
+    if (candidateCount > campaign.budget.maxCandidates) {
+        throw new Error(`experiment defines ${candidateCount} candidates but budget.maxCandidates is ${campaign.budget.maxCandidates}`);
+    }
+    return executeCampaignLifecycle(new CampaignExecution(execution, "experiment", runStaticCampaignWork));
+};
+export const optimizeExperimentCampaign = async (execution) => {
+    if (!execution.loaded.campaign.optimizer) {
+        throw new Error(`experiment '${execution.loaded.campaign.id}' has no optimizer configuration`);
+    }
+    return executeCampaignLifecycle(new CampaignExecution(execution, "optimize", runOptimizationCampaignWork));
+};
+// Internal exports for identity contract tests.
+export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };
+//# sourceMappingURL=experiment.js.map