@prismatic-io/lux 0.0.1

package/src/orchestrator/experiment.ts

@@ -0,0 +1,991 @@
+import { randomUUID } from "node:crypto";
+import { rm } from "node:fs/promises";
+import type { z } from "zod";
+import {
+  bootstrapMeanConfidenceInterval,
+  type Candidate,
+  type CandidateProposal,
+  type ExperimentCampaign,
+  type ExperimentSelector,
+  type Optimizer,
+} from "../core/index.js";
+import { createReflectiveOptimizer } from "../optimization/reflective.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
+import { type CampaignLifecycleEffects, executeCampaignLifecycle } from "./campaign-lifecycle.js";
+import {
+  captureSeedCandidate,
+  createCandidate,
+  hashSubjectTree,
+  materializeCandidate,
+  validateCandidate,
+} from "./candidates.js";
+import { selectCases } from "./discover.js";
+import { boundedExecutionSignal, ensureBudget, throwIfAborted } from "./experiment-budget.js";
+import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import {
+  type CampaignExecutionResult,
+  CampaignExecutionResultSchema,
+  type ExperimentCampaignPlan,
+  ExperimentCampaignPlanSchema,
+} from "./experiment-contracts.js";
+import {
+  agentCorpusHash,
+  assertGraderSubject,
+  graderCorpusHash,
+  prepareCorpus,
+  selectorsBySplit,
+} from "./experiment-corpus.js";
+import {
+  type EvaluationAccounting,
+  evaluateAgentCandidate,
+  evaluateOfflineGraderCandidate,
+  invokeWithVerifiedExecutionIdentity,
+} from "./experiment-evaluation.js";
+import { finishExperiment } from "./experiment-finish.js";
+import {
+  evaluationIdentity,
+  resolvedProfileSnapshot,
+  runtimeCliReferences,
+} from "./experiment-identity.js";
+import {
+  type ExperimentEvaluation,
+  loadExperimentCandidates,
+  loadExperimentRun,
+  type OpenedExperimentRun,
+  openExperimentRun,
+  resumeExperimentRun,
+} from "./experiment-runs.js";
+import {
+  claudeConfigRoot,
+  claudeUserBehaviorHash,
+  cliIdentity,
+  configuredClaudeSourceHashes,
+  LUX_VERSION,
+  runtimeCliIdentityHashForReferences,
+} from "./experiment-runtime-identity.js";
+import {
+  candidateEvaluations,
+  type ExperimentSelectionDerivationOptionsSchema,
+  rankCandidates,
+  searchFrontier,
+} from "./experiment-selection.js";
+import { loadEvalCase } from "./loader.js";
+import { executeOptimizationLifecycle } from "./optimization-lifecycle.js";
+
+export type { CampaignExecutionOptions } from "./experiment-context.js";
+export {
+  type CampaignExecutionResult,
+  CampaignExecutionResultSchema,
+  type ExperimentCampaignPlan,
+  ExperimentCampaignPlanSchema,
+} from "./experiment-contracts.js";
+export {
+  deriveExperimentSelection,
+  type ExperimentDecisionCandidate,
+  ExperimentDecisionCandidateSchema,
+  paretoFrontier,
+  rankCandidates,
+} from "./experiment-selection.js";
+
+type ExperimentSplit = "train" | "validation" | "test";
+const evaluateAcross = async (options: {
+  execution: CampaignExecutionOptions;
+  experiment: OpenedExperimentRun;
+  candidate: Candidate;
+  splits: Array<{ split: ExperimentSplit; selector: ExperimentSelector; repetitions: number }>;
+  corpus: EvaluationCorpus;
+  verifyExecutionIdentity: () => Promise<void>;
+}): Promise<ExperimentEvaluation[]> => {
+  const boundedSignal = boundedExecutionSignal(options.execution, options.experiment);
+  const boundedExecution = { ...options.execution, abortSignal: boundedSignal };
+  throwIfAborted(boundedSignal);
+  ensureBudget(options.execution.loaded.campaign, options.experiment);
+  const currentCorpusHash =
+    options.corpus.kind === "agent"
+      ? await agentCorpusHash(
+          options.execution,
+          options.corpus.cases,
+          selectorsBySplit(options.execution.loaded.campaign),
+        )
+      : await graderCorpusHash(
+          options.execution,
+          selectorsBySplit(options.execution.loaded.campaign),
+        );
+  if (currentCorpusHash !== options.corpus.hash) {
+    throw new Error("experiment evaluation corpus changed while the campaign was running");
+  }
+  const materializedRoot = await materializeCandidate(
+    options.execution.loaded.subjectRoot,
+    options.execution.loaded.campaign.subject,
+    options.candidate,
+  );
+  try {
+    await validateCandidate(
+      materializedRoot,
+      options.execution.loaded.campaign.subject,
+      boundedSignal,
+      options.verifyExecutionIdentity,
+    );
+    throwIfAborted(boundedSignal);
+    const evaluations: ExperimentEvaluation[] = [];
+    for (const profileId of options.execution.loaded.campaign.profiles) {
+      for (const split of options.splits) {
+        throwIfAborted(boundedSignal);
+        evaluations.push(
+          options.corpus.kind === "agent"
+            ? await evaluateAgentCandidate({
+                execution: boundedExecution,
+                experiment: options.experiment,
+                candidate: options.candidate,
+                allCases: options.corpus.cases,
+                profileId,
+                verifyExecutionIdentity: options.verifyExecutionIdentity,
+                ...split,
+              })
+            : await evaluateOfflineGraderCandidate({
+                execution: boundedExecution,
+                experiment: options.experiment,
+                candidate: options.candidate,
+                materializedRoot,
+                profileId,
+                verifyExecutionIdentity: options.verifyExecutionIdentity,
+                ...split,
+              }),
+        );
+        throwIfAborted(boundedSignal);
+        ensureBudget(options.execution.loaded.campaign, options.experiment);
+      }
+    }
+    const finalCorpusHash =
+      options.corpus.kind === "agent"
+        ? await agentCorpusHash(
+            options.execution,
+            options.corpus.cases,
+            selectorsBySplit(options.execution.loaded.campaign),
+          )
+        : await graderCorpusHash(
+            options.execution,
+            selectorsBySplit(options.execution.loaded.campaign),
+          );
+    if (finalCorpusHash !== options.corpus.hash) {
+      throw new Error(
+        "experiment evaluation corpus changed while candidate evaluation was running",
+      );
+    }
+    const finalSourceHash = await hashSubjectTree(
+      options.execution.loaded.subjectRoot,
+      options.execution.loaded.campaign.subject,
+    );
+    if (finalSourceHash !== options.candidate.baseSourceHash) {
+      throw new Error("experiment subject source changed while candidate evaluation was running");
+    }
+    return evaluations;
+  } finally {
+    await rm(materializedRoot, { recursive: true, force: true });
+  }
+};
+
+export const planExperimentCampaign = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+): Promise<ExperimentCampaignPlan> => {
+  const campaign = execution.loaded.campaign;
+  if (mode === "optimize") {
+    if (!campaign.optimizer) {
+      throw new Error(`experiment '${campaign.id}' has no optimizer configuration`);
+    }
+    createReflectiveOptimizer(campaign.optimizer.config);
+  }
+  const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
+  await assertGraderSubject(execution, baseline);
+  const corpus = await prepareCorpus(execution);
+  // Plan is the deterministic preflight for execution. Build the exact same
+  // identity now so profile, behavior-tree, and custom CLI failures happen
+  // before a user approves any model spend.
+  await evaluationIdentity(execution, corpus, mode);
+  const splitSelectors = selectorsBySplit(campaign);
+  const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
+  const splits = splitSelectors.map(({ name, selector }) => ({
+    split: name,
+    examples:
+      corpus.kind === "agent"
+        ? selectCases(corpus.cases, selector).length
+        : selectGraderAnnotations(graderAnnotations, selector).length,
+  }));
+  const profiles = campaign.profiles.length;
+  const count = (names: ExperimentSplit[]): number =>
+    splits
+      .filter((split) => names.includes(split.split))
+      .reduce((total, split) => total + split.examples, 0);
+  const projectedCandidates =
+    mode === "experiment" ? 1 + campaign.variants.length : campaign.budget.maxCandidates;
+  const projectedMetricCalls =
+    mode === "experiment"
+      ? projectedCandidates *
+        profiles *
+        count(["train", "validation", "test"]) *
+        campaign.repetitions.promotion
+      : projectedCandidates *
+          profiles *
+          count(["train", "validation"]) *
+          campaign.repetitions.search +
+        (1 + (campaign.optimizer?.promotionTopK ?? 1)) *
+          profiles *
+          count(["validation", "test"]) *
+          campaign.repetitions.promotion;
+  let maxRubricAssertions = 0;
+  if (corpus.kind === "grader") {
+    // Grader candidates may grow any rubric to the loader's hard cap.
+    maxRubricAssertions = 20;
+  } else {
+    const selected = splitSelectors.flatMap((split) => selectCases(corpus.cases, split.selector));
+    maxRubricAssertions = Math.max(
+      0,
+      ...(await Promise.all(selected.map((evalCase) => loadEvalCase(evalCase.path)))).map(
+        (evalCase) =>
+          (evalCase.assertions ?? []).filter((assertion) => assertion.type === "rubric").length,
+      ),
+    );
+  }
+  const projectedJudgeCalls = projectedMetricCalls * maxRubricAssertions * 2;
+  const projectedProposalCalls = mode === "optimize" ? campaign.budget.maxProposalCalls : 0;
+  const warnings = campaign.objectives.constraints.flatMap((constraint) => {
+    if (!constraint.confidence) return [];
+    const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
+    let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
+    if (corpus.kind === "grader" && splitSelector) {
+      const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+      if (constraint.metric === "grader.macroFalsePositiveRate") {
+        examples = selected.filter((annotation) =>
+          [
+            annotation.expected.casePassed,
+            ...Object.values(annotation.expected.assertions).map((label) => label.passed),
+          ].includes(false),
+        ).length;
+      } else if (constraint.metric === "grader.macroFalseNegativeRate") {
+        examples = selected.filter((annotation) =>
+          [
+            annotation.expected.casePassed,
+            ...Object.values(annotation.expected.assertions).map((label) => label.passed),
+          ].includes(true),
+        ).length;
+      } else if (constraint.metric === "grader.scoreMae") {
+        examples = selected.filter(
+          (annotation) =>
+            annotation.expected.caseScore !== undefined ||
+            Object.values(annotation.expected.assertions).some(
+              (label) => label.score !== undefined,
+            ),
+        ).length;
+      } else if (constraint.metric === "grader.caseAgreement") {
+        examples = selected.filter(
+          (annotation) =>
+            annotation.expected.casePassed !== undefined ||
+            annotation.expected.caseScore !== undefined,
+        ).length;
+      } else if (constraint.metric === "grader.assertionAgreement") {
+        examples = selected.filter(
+          (annotation) => Object.keys(annotation.expected.assertions).length > 0,
+        ).length;
+      } else if (
+        constraint.metric === "grader.abstentionRate" ||
+        constraint.metric === "grader.coverage"
+      ) {
+        examples = selected.filter(
+          (annotation) =>
+            annotation.expected.casePassed !== undefined ||
+            Object.keys(annotation.expected.assertions).length > 0,
+        ).length;
+      }
+    }
+    if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
+      examples = 1;
+    }
+    const notes: string[] = [];
+    if (examples < constraint.confidence.minSamples) {
+      notes.push(
+        `${constraint.split} metric '${constraint.metric}' has ${examples} independent samples, below confidence minSamples=${constraint.confidence.minSamples}`,
+      );
+    } else if (
+      constraint.confidence.method === "wilson" ||
+      constraint.confidence.method === "hoeffding"
+    ) {
+      const bestForMin = bootstrapMeanConfidenceInterval(
+        Array<number>(examples).fill(1),
+        constraint.confidence,
+      );
+      const bestForMax = bootstrapMeanConfidenceInterval(
+        Array<number>(examples).fill(0),
+        constraint.confidence,
+      );
+      if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
+        notes.push(
+          `${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`,
+        );
+      }
+      if (constraint.max !== undefined && bestForMax.upper > constraint.max) {
+        notes.push(
+          `${constraint.split} metric '${constraint.metric}' cannot attain confidence upper bound ${constraint.max} with only ${examples} samples (best ${bestForMax.upper.toFixed(3)})`,
+        );
+      }
+    }
+    return notes;
+  });
+  warnings.push(
+    "token, reported-cost, and wall-time limits are enforced from observed runtime usage and cannot be proven by the no-call plan",
+  );
+  return ExperimentCampaignPlanSchema.parse({
+    campaignId: campaign.id,
+    mode,
+    evaluationKind: campaign.evaluation.kind,
+    subjectRoot: execution.loaded.subjectRoot,
+    mutableComponents: Object.keys(baseline.components).sort(),
+    profiles: [...campaign.profiles],
+    splits,
+    projected: {
+      candidates: projectedCandidates,
+      metricCalls: projectedMetricCalls,
+      judgeCalls: projectedJudgeCalls,
+      proposalCalls: projectedProposalCalls,
+    },
+    budget: campaign.budget,
+    fitsProjectedCallBudgets:
+      projectedCandidates <= campaign.budget.maxCandidates &&
+      projectedMetricCalls <= campaign.budget.maxMetricCalls &&
+      projectedJudgeCalls <= campaign.budget.maxJudgeCalls &&
+      projectedProposalCalls <= campaign.budget.maxProposalCalls,
+    warnings,
+  });
+};
+
+const prepareExecution = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+): Promise<{
+  baseline: Candidate;
+  candidates: Candidate[];
+  experiment: OpenedExperimentRun;
+  corpus: EvaluationCorpus;
+  verifyExecutionIdentity: () => Promise<void>;
+}> => {
+  const campaign = execution.loaded.campaign;
+  const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
+  await assertGraderSubject(execution, baseline);
+  const corpus = await prepareCorpus(execution);
+  const runtimeReferences = await runtimeCliReferences(execution, corpus, mode);
+  const expectedRuntimeCliIdentityHash =
+    await runtimeCliIdentityHashForReferences(runtimeReferences);
+  const currentEvaluationHash = await evaluationIdentity(
+    execution,
+    corpus,
+    mode,
+    expectedRuntimeCliIdentityHash,
+  );
+  const verifyExecutionIdentity = async (): Promise<void> => {
+    const observed = await runtimeCliIdentityHashForReferences(runtimeReferences);
+    if (observed !== expectedRuntimeCliIdentityHash) {
+      throw new Error("experiment CLI identity changed during execution");
+    }
+  };
+  await verifyExecutionIdentity();
+  if (execution.resumeFrom) {
+    const saved = await loadExperimentRun(execution.resumeFrom);
+    if (!saved) {
+      throw new Error(`experiment manifest not found or invalid at ${execution.resumeFrom}`);
+    }
+    if (saved.mode !== mode) {
+      throw new Error(`cannot resume ${saved.mode} run with lux ${mode}`);
+    }
+    if (saved.campaignId !== campaign.id || saved.campaignHash !== execution.loaded.campaignHash) {
+      throw new Error("cannot resume: the experiment campaign definition changed");
+    }
+    if (saved.evaluationKind !== campaign.evaluation.kind) {
+      throw new Error("cannot resume: the experiment evaluation kind changed");
+    }
+    if (saved.evaluationHash !== currentEvaluationHash) {
+      throw new Error("cannot resume: the evaluation corpus or profile configuration changed");
+    }
+    if (
+      saved.subjectRoot !== execution.loaded.subjectRoot ||
+      saved.baseSourceHash !== baseline.baseSourceHash
+    ) {
+      throw new Error("cannot resume: the experiment subject source changed");
+    }
+    const experiment = await resumeExperimentRun(execution.resumeFrom);
+    try {
+      const candidates = await loadExperimentCandidates(experiment.path);
+      const savedBaseline = candidates.find((candidate) => candidate.id === "baseline");
+      if (!savedBaseline) throw new Error("cannot resume: baseline candidate is missing");
+      return {
+        baseline: savedBaseline,
+        candidates,
+        experiment,
+        corpus,
+        verifyExecutionIdentity,
+      };
+    } catch (error) {
+      await experiment.release();
+      throw error;
+    }
+  }
+  const experiment = await openExperimentRun({
+    runsRoot: execution.runsRoot ?? execution.config.runsRoot,
+    campaignId: campaign.id,
+    campaignPath: execution.loaded.path,
+    campaignHash: execution.loaded.campaignHash,
+    campaignSnapshot: campaign,
+    resolvedProfiles: await resolvedProfileSnapshot(execution),
+    luxVersion: LUX_VERSION,
+    subjectRoot: execution.loaded.subjectRoot,
+    baseSourceHash: baseline.baseSourceHash,
+    mode,
+    evaluationKind: campaign.evaluation.kind,
+    evaluationHash: currentEvaluationHash,
+  });
+  try {
+    await experiment.saveCandidate(baseline);
+    return {
+      baseline,
+      candidates: [baseline],
+      experiment,
+      corpus,
+      verifyExecutionIdentity,
+    };
+  } catch (error) {
+    await experiment.release();
+    throw error;
+  }
+};
+
+const splitPlan = (
+  campaign: ExperimentCampaign,
+  repetitions: number,
+  includeTrain: boolean,
+): Array<{ split: ExperimentSplit; selector: ExperimentSelector; repetitions: number }> => [
+  ...(includeTrain && campaign.splits.train
+    ? [{ split: "train" as const, selector: campaign.splits.train, repetitions }]
+    : []),
+  { split: "validation", selector: campaign.splits.validation, repetitions },
+  ...(campaign.splits.test
+    ? [{ split: "test" as const, selector: campaign.splits.test, repetitions }]
+    : []),
+];
+
+type PreparedCampaignExecution = Awaited<ReturnType<typeof prepareExecution>>;
+type CampaignWork = {
+  candidates: Candidate[];
+  selectionOptions?: z.input<typeof ExperimentSelectionDerivationOptionsSchema>;
+};
+type CampaignWorker = (
+  execution: CampaignExecutionOptions,
+  prepared: PreparedCampaignExecution,
+) => Promise<CampaignWork>;
+
+class CampaignExecution implements CampaignLifecycleEffects<CampaignWork, CampaignExecutionResult> {
+  private prepared: PreparedCampaignExecution | undefined;
+  private readonly execution: CampaignExecutionOptions;
+  private readonly mode: "experiment" | "optimize";
+  private readonly worker: CampaignWorker;
+
+  constructor(
+    execution: CampaignExecutionOptions,
+    mode: "experiment" | "optimize",
+    worker: CampaignWorker,
+  ) {
+    this.execution = execution;
+    this.mode = mode;
+    this.worker = worker;
+  }
+
+  async prepare(): Promise<void> {
+    this.prepared = await prepareExecution(this.execution, this.mode);
+  }
+
+  execute(): Promise<CampaignWork> {
+    return this.worker(this.execution, this.requirePrepared());
+  }
+
+  async complete(work: CampaignWork): Promise<CampaignExecutionResult> {
+    const prepared = this.requirePrepared();
+    const champion = await finishExperiment(
+      prepared.experiment,
+      this.execution.loaded.campaign,
+      work.candidates,
+      work.selectionOptions,
+    );
+    return CampaignExecutionResultSchema.parse({
+      experimentDir: prepared.experiment.path,
+      champion,
+      manifest: prepared.experiment.manifest,
+    });
+  }
+
+  resultStatus(result: CampaignExecutionResult): "passed" | "failed" {
+    return result.manifest.status === "passed" ? "passed" : "failed";
+  }
+
+  failureStatus(_error: unknown): "failed" | "aborted" {
+    return this.execution.abortSignal?.aborted ? "aborted" : "failed";
+  }
+
+  async recordFailure(status: "failed" | "aborted", error: unknown): Promise<void> {
+    if (!this.prepared) return;
+    const { manifest } = this.prepared.experiment;
+    manifest.status = status;
+    manifest.endedAt = new Date().toISOString();
+    manifest.errorMessage = error instanceof Error ? error.message : String(error);
+    await this.prepared.experiment.write();
+  }
+
+  async close(_primaryError: unknown | null): Promise<void> {
+    await this.prepared?.experiment.release();
+  }
+
+  private requirePrepared(): PreparedCampaignExecution {
+    if (!this.prepared) throw new Error("campaign lifecycle entered execution before preparation");
+    return this.prepared;
+  }
+}
+
+const runStaticCampaignWork = async (
+  execution: CampaignExecutionOptions,
+  prepared: PreparedCampaignExecution,
+): Promise<CampaignWork> => {
+  const campaign = execution.loaded.campaign;
+  const {
+    baseline,
+    candidates: savedCandidates,
+    experiment,
+    corpus,
+    verifyExecutionIdentity,
+  } = prepared;
+  const candidates = [
+    baseline,
+    ...campaign.variants.map((variant) => {
+      const saved = savedCandidates.find((savedCandidate) => savedCandidate.id === variant.id);
+      if (saved) return saved;
+      const candidate = createCandidate(
+        baseline,
+        { updates: variant.components },
+        campaign.subject,
+        { id: variant.id, proposer: "manual" },
+      );
+      candidate.parentIds = variant.parentIds.length > 0 ? variant.parentIds : [baseline.id];
+      return candidate;
+    }),
+  ];
+  for (const candidate of candidates) {
+    throwIfAborted(execution.abortSignal);
+    await experiment.saveCandidate(candidate);
+    await evaluateAcross({
+      execution,
+      experiment,
+      candidate,
+      corpus,
+      verifyExecutionIdentity,
+      splits: splitPlan(campaign, campaign.repetitions.promotion, true),
+    });
+  }
+  return { candidates };
+};
+
+type OptimizationContext = {
+  execution: CampaignExecutionOptions;
+  prepared: PreparedCampaignExecution;
+  optimizer: Optimizer;
+  configuration: NonNullable<ExperimentCampaign["optimizer"]>;
+  searchSplits: ReturnType<typeof splitPlan>;
+};
+
+type OptimizationSearchState = {
+  readonly iteration: number;
+  readonly staleIterations: number;
+  readonly bestId: string;
+};
+
+type OptimizationIterationPlan = {
+  readonly current: Candidate;
+  readonly alternatives: readonly Candidate[];
+  readonly frontierHashes: ReadonlySet<string>;
+};
+
+type OptimizationPromotionPlan = {
+  readonly finalists: readonly Candidate[];
+};
+
+type OptimizationDeploymentPlan = {
+  readonly deployable: readonly Candidate[];
+  readonly finalistIds: readonly string[];
+};
+
+const createOptimizationContext = (
+  execution: CampaignExecutionOptions,
+  prepared: PreparedCampaignExecution,
+): OptimizationContext => {
+  const campaign = execution.loaded.campaign;
+  const configuration = campaign.optimizer;
+  if (!configuration) {
+    throw new Error(`experiment '${campaign.id}' has no optimizer configuration`);
+  }
+  return {
+    execution,
+    prepared,
+    configuration,
+    optimizer: createReflectiveOptimizer(configuration.config),
+    searchSplits: splitPlan(campaign, campaign.repetitions.search, true).filter(
+      (entry) => entry.split !== "test",
+    ),
+  };
+};
+
+const rankedSearchCandidates = (
+  context: OptimizationContext,
+  candidates: readonly Candidate[],
+): Candidate[] =>
+  rankCandidates(
+    context.execution.loaded.campaign,
+    candidates,
+    context.prepared.experiment.manifest,
+    ["train", "validation"],
+  );
+
+const optimizationFrontier = (
+  context: OptimizationContext,
+  candidates: Candidate[],
+): Candidate[] => {
+  if (context.configuration.candidateSelection === "pareto") {
+    return searchFrontier(
+      context.execution.loaded.campaign,
+      candidates,
+      context.prepared.experiment.manifest,
+    );
+  }
+  return [rankedSearchCandidates(context, candidates)[0] ?? context.prepared.baseline];
+};
+
+const alternativeCandidates = (
+  context: OptimizationContext,
+  frontier: readonly Candidate[],
+  current: Candidate,
+): Candidate[] =>
+  context.configuration.mergeFrontier
+    ? frontier.filter((candidate) => candidate.id !== current.id).slice(0, 2)
+    : [];
+
+const trainEvaluations = (
+  experiment: OpenedExperimentRun,
+  candidateId: string,
+): ExperimentEvaluation[] =>
+  candidateEvaluations(experiment.manifest, candidateId).filter(
+    (evaluation) => evaluation.split === "train",
+  );
+
+const requestOptimizationProposals = async (
+  context: OptimizationContext,
+  state: OptimizationSearchState,
+  current: Candidate,
+  alternatives: Candidate[],
+): Promise<CandidateProposal[]> => {
+  const { execution, prepared, optimizer } = context;
+  const { experiment, verifyExecutionIdentity, candidates } = prepared;
+  const campaign = execution.loaded.campaign;
+  const proposalSignal = boundedExecutionSignal(execution, experiment);
+  const proposalAttemptIds: string[] = [];
+  let pendingAccounting: EvaluationAccounting | undefined;
+
+  const input: Parameters<Optimizer["propose"]>[0] = {
+    campaign,
+    candidate: current,
+    observations: trainEvaluations(experiment, current.id),
+    remainingCandidates: campaign.budget.maxCandidates - candidates.length,
+    cwd: execution.loaded.directory,
+    abortSignal: proposalSignal,
+    recordModelAttemptStart: async (kind) => {
+      await verifyExecutionIdentity();
+      proposalAttemptIds.push(await experiment.startModelAttempt(kind));
+    },
+    completeModelAttempts: async () => {
+      await verifyExecutionIdentity();
+      if (pendingAccounting) {
+        await experiment.settleModelAttempts(
+          `proposal-spend-${randomUUID()}`,
+          pendingAccounting,
+          proposalAttemptIds,
+        );
+        pendingAccounting = undefined;
+      } else {
+        await experiment.completeModelAttempts(proposalAttemptIds);
+      }
+      proposalAttemptIds.length = 0;
+      ensureBudget(campaign, experiment);
+    },
+    recordModelCall: async (accounting) => {
+      await verifyExecutionIdentity();
+      pendingAccounting = {
+        usage: accounting.usage,
+        costUsd: accounting.costUsd ?? 0,
+        unpricedModelCalls: accounting.costUsd === undefined ? 1 : 0,
+      };
+    },
+  };
+  if (alternatives.length > 0) {
+    input.alternatives = alternatives;
+    input.alternativeObservations = Object.fromEntries(
+      alternatives.map((candidate) => [candidate.id, trainEvaluations(experiment, candidate.id)]),
+    );
+  }
+
+  const proposals = await invokeWithVerifiedExecutionIdentity(verifyExecutionIdentity, async () => {
+    ensureBudget(campaign, experiment, { proposalCalls: 1 });
+    await experiment.recordAccounting(`proposal:${state.iteration}:${randomUUID()}`, {
+      proposalCalls: 1,
+    });
+    await experiment.write();
+    return optimizer.propose(input);
+  });
+  throwIfAborted(proposalSignal);
+  ensureBudget(campaign, experiment);
+  return proposals;
+};
+
+const addOptimizationCandidates = async (
+  context: OptimizationContext,
+  current: Candidate,
+  alternatives: readonly Candidate[],
+  proposals: readonly CandidateProposal[],
+): Promise<number> => {
+  const { execution, prepared, configuration, searchSplits } = context;
+  const { candidates, experiment, corpus, verifyExecutionIdentity } = prepared;
+  const campaign = execution.loaded.campaign;
+  let added = 0;
+  for (const proposal of proposals) {
+    const candidate = createCandidate(current, proposal, campaign.subject, {
+      proposer: `reflective:${configuration.algorithm}`,
+    });
+    if (alternatives.length > 0) {
+      candidate.parentIds = [current.id, ...alternatives.map((entry) => entry.id)];
+    }
+    if (candidates.some((entry) => entry.candidateHash === candidate.candidateHash)) continue;
+    candidates.push(candidate);
+    added++;
+    await experiment.saveCandidate(candidate);
+    await evaluateAcross({
+      execution,
+      experiment,
+      candidate,
+      corpus,
+      verifyExecutionIdentity,
+      splits: searchSplits,
+    });
+    if (candidates.length >= campaign.budget.maxCandidates) break;
+  }
+  return added;
+};
+
+const advanceSearchState = (
+  context: OptimizationContext,
+  state: OptimizationSearchState,
+  added: number,
+  previousFrontier: ReadonlySet<string>,
+): OptimizationSearchState => {
+  const { candidates, baseline } = context.prepared;
+  const nextBest = rankedSearchCandidates(context, candidates)[0]?.id ?? baseline.id;
+  const expandedFrontier =
+    context.configuration.candidateSelection === "pareto" &&
+    optimizationFrontier(context, candidates).some(
+      (candidate) => !previousFrontier.has(candidate.candidateHash),
+    );
+  return {
+    iteration: state.iteration + 1,
+    staleIterations:
+      added > 0 && (nextBest !== state.bestId || expandedFrontier) ? 0 : state.staleIterations + 1,
+    bestId: nextBest,
+  };
+};
+
+const planOptimizationIteration = (
+  context: OptimizationContext,
+  state: OptimizationSearchState,
+): OptimizationIterationPlan => {
+  throwIfAborted(context.execution.abortSignal);
+  const frontier = optimizationFrontier(context, context.prepared.candidates);
+  const current = frontier[state.iteration % frontier.length] ?? context.prepared.baseline;
+  return {
+    current,
+    alternatives: alternativeCandidates(context, frontier, current),
+    frontierHashes: new Set(frontier.map((candidate) => candidate.candidateHash)),
+  };
+};
+
+const searchCanContinue = (
+  context: OptimizationContext,
+  state: OptimizationSearchState,
+): boolean => {
+  const { budget } = context.execution.loaded.campaign;
+  return (
+    context.prepared.candidates.length < budget.maxCandidates &&
+    state.staleIterations < budget.patience &&
+    context.prepared.experiment.manifest.budgetUsage.proposalCalls < budget.maxProposalCalls &&
+    !context.execution.abortSignal?.aborted
+  );
+};
+
+const evaluateSearchSeeds = async (context: OptimizationContext): Promise<void> => {
+  const { prepared, searchSplits } = context;
+  for (const candidate of prepared.candidates) {
+    await evaluateAcross({
+      execution: context.execution,
+      experiment: prepared.experiment,
+      candidate,
+      corpus: prepared.corpus,
+      verifyExecutionIdentity: prepared.verifyExecutionIdentity,
+      splits: searchSplits,
+    });
+  }
+  throwIfAborted(context.execution.abortSignal);
+};
+
+const initialOptimizationSearch = (context: OptimizationContext): OptimizationSearchState => ({
+  iteration: 0,
+  staleIterations: 0,
+  bestId:
+    rankedSearchCandidates(context, context.prepared.candidates)[0]?.id ??
+    context.prepared.baseline.id,
+});
+
+const evaluateOptimizationProposals = async (
+  context: OptimizationContext,
+  state: OptimizationSearchState,
+  plan: OptimizationIterationPlan,
+  proposals: readonly CandidateProposal[],
+): Promise<OptimizationSearchState> => {
+  const added = await addOptimizationCandidates(
+    context,
+    plan.current,
+    plan.alternatives,
+    proposals,
+  );
+  return advanceSearchState(context, state, added, plan.frontierHashes);
+};
+
+const selectPromotionCandidates = (context: OptimizationContext): OptimizationPromotionPlan => {
+  const { prepared, configuration } = context;
+  const { baseline, candidates } = prepared;
+  const searchLeaders = rankedSearchCandidates(context, candidates)
+    .filter((candidate) => candidate.id !== baseline.id)
+    .slice(0, configuration.promotionTopK);
+  return { finalists: [baseline, ...searchLeaders] };
+};
+
+const evaluatePromotionCandidates = async (
+  context: OptimizationContext,
+  plan: OptimizationPromotionPlan,
+): Promise<OptimizationDeploymentPlan> => {
+  const { execution, prepared } = context;
+  const { baseline, experiment, corpus, verifyExecutionIdentity } = prepared;
+  const campaign = execution.loaded.campaign;
+  for (const candidate of plan.finalists) {
+    await evaluateAcross({
+      execution,
+      experiment,
+      candidate,
+      corpus,
+      verifyExecutionIdentity,
+      splits: [
+        {
+          split: "validation",
+          selector: campaign.splits.validation,
+          repetitions: campaign.repetitions.promotion,
+        },
+      ],
+    });
+  }
+  const promoted = rankedSearchCandidates(context, plan.finalists)[0] ?? baseline;
+  return {
+    deployable: promoted.id === baseline.id ? [baseline] : [baseline, promoted],
+    finalistIds: plan.finalists.map((candidate) => candidate.id),
+  };
+};
+
+const evaluateDeploymentCandidates = async (
+  context: OptimizationContext,
+  plan: OptimizationDeploymentPlan,
+): Promise<CampaignWork> => {
+  const { execution, prepared } = context;
+  const { experiment, corpus, verifyExecutionIdentity } = prepared;
+  const campaign = execution.loaded.campaign;
+  if (campaign.splits.test) {
+    for (const candidate of plan.deployable) {
+      await evaluateAcross({
+        execution,
+        experiment,
+        candidate,
+        corpus,
+        verifyExecutionIdentity,
+        splits: [
+          {
+            split: "test",
+            selector: campaign.splits.test,
+            repetitions: campaign.repetitions.promotion,
+          },
+        ],
+      });
+    }
+  }
+  return {
+    candidates: [...plan.deployable],
+    selectionOptions: {
+      promotionFinalistCandidateIds: [...plan.finalistIds],
+    },
+  };
+};
+
+const runOptimizationCampaignWork = async (
+  execution: CampaignExecutionOptions,
+  prepared: PreparedCampaignExecution,
+): Promise<CampaignWork> => {
+  const context = createOptimizationContext(execution, prepared);
+  return executeOptimizationLifecycle({
+    initialSearch: () => initialOptimizationSearch(context),
+    evaluateSearchSeeds: () => evaluateSearchSeeds(context),
+    searchCanContinue: (search) => searchCanContinue(context, search),
+    runSearchIteration: async (search) => {
+      const plan = planOptimizationIteration(context, search);
+      const proposals = await requestOptimizationProposals(context, search, plan.current, [
+        ...plan.alternatives,
+      ]);
+      return evaluateOptimizationProposals(context, search, plan, proposals);
+    },
+    promote: async () => evaluatePromotionCandidates(context, selectPromotionCandidates(context)),
+    test: (deployment) => evaluateDeploymentCandidates(context, deployment),
+  });
+};
+
+export const runExperimentCampaign = async (
+  execution: CampaignExecutionOptions,
+): Promise<CampaignExecutionResult> => {
+  const campaign = execution.loaded.campaign;
+  const candidateCount = 1 + campaign.variants.length;
+  if (candidateCount > campaign.budget.maxCandidates) {
+    throw new Error(
+      `experiment defines ${candidateCount} candidates but budget.maxCandidates is ${campaign.budget.maxCandidates}`,
+    );
+  }
+  return executeCampaignLifecycle(
+    new CampaignExecution(execution, "experiment", runStaticCampaignWork),
+  );
+};
+
+export const optimizeExperimentCampaign = async (
+  execution: CampaignExecutionOptions,
+): Promise<CampaignExecutionResult> => {
+  if (!execution.loaded.campaign.optimizer) {
+    throw new Error(`experiment '${execution.loaded.campaign.id}' has no optimizer configuration`);
+  }
+  return executeCampaignLifecycle(
+    new CampaignExecution(execution, "optimize", runOptimizationCampaignWork),
+  );
+};
+
+// Internal exports for identity contract tests.
+export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };