@prismatic-io/lux 0.0.1

package/src/orchestrator/driver-session.ts

@@ -0,0 +1,507 @@
+import { assign, createActor, fromPromise, setup, toPromise } from "xstate";
+import {
+  type AgentDriver,
+  type Answer,
+  type Answerer,
+  type AnswererContext,
+  type AnswerHistoryEntry,
+  AnswerSchema,
+  type Done,
+  type ErrorEvent,
+  type Interrupt,
+  type Progress,
+  type RunEvent,
+} from "../core/index.js";
+import type { OpenedRunDir } from "./run-dir.js";
+import type { RunTraceLimitFailure } from "./trace-limits.js";
+
+export type DriverSessionOutcome =
+  | { kind: "done"; done: Done }
+  | {
+      kind: "error";
+      error: ErrorEvent;
+      operationalFailure?: {
+        phase: "answerer" | "driver-transport" | "trace" | "abort";
+        reason: string;
+      };
+    };
+
+export type DriverSession = {
+  outcome: Promise<DriverSessionOutcome>;
+  fail: (reason: string) => void;
+};
+
+type DeferredInterrupt = {
+  interrupt: Interrupt;
+  complete: () => void;
+};
+
+type CompletedAnswer = {
+  answer: Answer;
+  durationMs: number;
+};
+
+export type DriverSessionEffects = {
+  answer: (interrupt: Interrupt) => Promise<CompletedAnswer>;
+  respond: (interrupt: Interrupt, completed: CompletedAnswer) => Promise<void>;
+  progress: (progress: Progress) => void;
+  terminal: (outcome: DriverSessionOutcome, alreadyRecorded: boolean) => void;
+};
+
+type DriverSessionContext = {
+  effects: DriverSessionEffects;
+  current: DeferredInterrupt | null;
+  queued: DeferredInterrupt[];
+  terminal: DriverSessionOutcome | null;
+  completedAnswer: CompletedAnswer | null;
+};
+
+type DriverSessionEvent =
+  | { type: "interrupt"; pending: DeferredInterrupt }
+  | { type: "progress"; progress: Progress }
+  | { type: "done"; done: Done; recorded?: boolean }
+  | { type: "error"; error: ErrorEvent; recorded?: boolean }
+  | { type: "abort"; recorded?: boolean }
+  | { type: "fail"; reason: string; recorded?: boolean }
+  | { type: "traceLimit"; failure: RunTraceLimitFailure };
+type RecordableTerminalEvent = Extract<
+  DriverSessionEvent,
+  { type: "done" | "error" | "abort" | "fail" }
+>;
+
+class InteractionFailure extends Error {
+  readonly outcome: DriverSessionOutcome;
+  readonly terminalRecorded: boolean;
+
+  constructor(outcome: DriverSessionOutcome, terminalRecorded = false) {
+    super(outcome.kind === "error" ? outcome.error.reason : "driver interaction failed");
+    this.outcome = outcome;
+    this.terminalRecorded = terminalRecorded;
+  }
+}
+
+const interactionFailure = (error: unknown): DriverSessionOutcome =>
+  error instanceof InteractionFailure
+    ? error.outcome
+    : {
+        kind: "error",
+        error: { exitReason: "error", reason: "driver interaction failed" },
+      };
+
+const doneOutcome = (event: DriverSessionEvent): DriverSessionOutcome =>
+  event.type === "done"
+    ? { kind: "done", done: event.done }
+    : {
+        kind: "error",
+        error: { exitReason: "error", reason: "invalid driver completion event" },
+      };
+
+const errorOutcome = (event: DriverSessionEvent): DriverSessionOutcome => {
+  if (event.type === "error") return { kind: "error", error: event.error };
+  const reason = event.type === "fail" ? event.reason : "run aborted";
+  return {
+    kind: "error",
+    error: { exitReason: "error", reason },
+    ...(event.type === "abort" ? { operationalFailure: { phase: "abort" as const, reason } } : {}),
+  };
+};
+
+const traceLimitOutcome = (event: DriverSessionEvent): DriverSessionOutcome => {
+  const reason =
+    event.type === "traceLimit" ? event.failure.reason : "run trace exceeded its limit";
+  return {
+    kind: "error",
+    error: { exitReason: "error", reason },
+    operationalFailure: { phase: "trace", reason },
+  };
+};
+
+const driverSessionMachine = setup({
+  types: {
+    context: {} as DriverSessionContext,
+    input: {} as { effects: DriverSessionEffects },
+    events: {} as DriverSessionEvent,
+    output: {} as DriverSessionOutcome,
+  },
+  actors: {
+    answer: fromPromise<CompletedAnswer, { effects: DriverSessionEffects; interrupt: Interrupt }>(
+      ({ input }) => input.effects.answer(input.interrupt),
+    ),
+    respond: fromPromise<
+      void,
+      {
+        effects: DriverSessionEffects;
+        interrupt: Interrupt;
+        completed: CompletedAnswer;
+      }
+    >(({ input }) => input.effects.respond(input.interrupt, input.completed)),
+  },
+  guards: {
+    hasNoTerminal: ({ context }) => context.terminal === null,
+    hasDoneTerminal: ({ context }) => context.terminal?.kind === "done",
+    hasErrorTerminal: ({ context }) => context.terminal?.kind === "error",
+    hasTraceTerminal: ({ context }) =>
+      context.terminal?.kind === "error" && context.terminal.operationalFailure?.phase === "trace",
+    hasQueuedInterrupt: ({ context }) => context.queued.length > 0,
+  },
+  actions: {
+    beginInterrupt: assign({
+      current: ({ event }) => (event.type === "interrupt" ? event.pending : null),
+    }),
+    queueInterrupt: assign({
+      queued: ({ context, event }) =>
+        event.type === "interrupt" ? [...context.queued, event.pending] : context.queued,
+    }),
+    takeNextInterrupt: assign(({ context }) => ({
+      current: context.queued[0] ?? null,
+      queued: context.queued.slice(1),
+    })),
+    completeCurrentInterrupt: assign(({ context }) => {
+      context.current?.complete();
+      return { current: null, completedAnswer: null };
+    }),
+    releaseInterrupts: ({ context }) => {
+      context.current?.complete();
+      for (const pending of context.queued) pending.complete();
+    },
+    recordProgress: ({ context, event }) => {
+      if (event.type === "progress") context.effects.progress(event.progress);
+    },
+    recordDone: ({ context, event }) => {
+      context.effects.terminal(
+        doneOutcome(event),
+        event.type === "done" && event.recorded === true,
+      );
+    },
+    rememberDone: assign({
+      terminal: ({ event }) => doneOutcome(event),
+    }),
+    recordError: ({ context, event }) => {
+      context.effects.terminal(errorOutcome(event), "recorded" in event && event.recorded === true);
+    },
+    rememberError: assign({
+      terminal: ({ event }) => errorOutcome(event),
+    }),
+    recordTraceLimit: ({ context, event }) => {
+      context.effects.terminal(traceLimitOutcome(event), true);
+    },
+    rememberTraceLimit: assign({
+      terminal: ({ event }) => traceLimitOutcome(event),
+    }),
+    recordInteractionFailure: ({ context, event }) => {
+      if ("error" in event) {
+        context.effects.terminal(
+          interactionFailure(event.error),
+          event.error instanceof InteractionFailure && event.error.terminalRecorded,
+        );
+      }
+    },
+    rememberInteractionFailure: assign({
+      terminal: ({ event }) =>
+        "error" in event ? interactionFailure(event.error) : errorOutcome(event),
+    }),
+    rememberCompletedAnswer: assign({
+      completedAnswer: ({ event }) =>
+        "output" in event ? (event.output as CompletedAnswer) : null,
+    }),
+  },
+}).createMachine({
+  id: "lux-driver-session",
+  context: ({ input }) => ({
+    effects: input.effects,
+    current: null,
+    queued: [],
+    terminal: null,
+    completedAnswer: null,
+  }),
+  initial: "waiting",
+  states: {
+    waiting: {
+      on: {
+        interrupt: { target: "answering", actions: "beginInterrupt" },
+        progress: { actions: "recordProgress" },
+        done: { target: "succeeded", actions: ["recordDone", "rememberDone"] },
+        error: { target: "failed", actions: ["recordError", "rememberError"] },
+        abort: { target: "failed", actions: ["recordError", "rememberError"] },
+        fail: { target: "failed", actions: ["recordError", "rememberError"] },
+        traceLimit: {
+          target: "failed",
+          actions: ["recordTraceLimit", "rememberTraceLimit"],
+        },
+      },
+    },
+    answering: {
+      invoke: {
+        src: "answer",
+        input: ({ context }) => {
+          if (!context.current) throw new Error("driver session has no current interrupt");
+          return { effects: context.effects, interrupt: context.current.interrupt };
+        },
+        onDone: [
+          {
+            guard: ({ context }) => context.terminal !== null,
+            target: "selecting",
+            actions: "completeCurrentInterrupt",
+          },
+          { target: "responding", actions: "rememberCompletedAnswer" },
+        ],
+        onError: [
+          {
+            guard: "hasTraceTerminal",
+            target: "selecting",
+            actions: "completeCurrentInterrupt",
+          },
+          {
+            target: "failed",
+            actions: ["recordInteractionFailure", "rememberInteractionFailure"],
+          },
+        ],
+      },
+      on: {
+        interrupt: { actions: "queueInterrupt" },
+        progress: { actions: "recordProgress" },
+        done: {
+          guard: "hasNoTerminal",
+          actions: ["recordDone", "rememberDone"],
+        },
+        error: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        abort: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        fail: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        traceLimit: {
+          actions: ["recordTraceLimit", "rememberTraceLimit"],
+        },
+      },
+    },
+    responding: {
+      invoke: {
+        src: "respond",
+        input: ({ context }) => {
+          if (!context.current || !context.completedAnswer) {
+            throw new Error("driver session has no completed answer to send");
+          }
+          return {
+            effects: context.effects,
+            interrupt: context.current.interrupt,
+            completed: context.completedAnswer,
+          };
+        },
+        onDone: { target: "selecting", actions: "completeCurrentInterrupt" },
+        onError: [
+          {
+            guard: "hasTraceTerminal",
+            target: "selecting",
+            actions: "completeCurrentInterrupt",
+          },
+          {
+            target: "failed",
+            actions: ["recordInteractionFailure", "rememberInteractionFailure"],
+          },
+        ],
+      },
+      on: {
+        interrupt: { actions: "queueInterrupt" },
+        progress: { actions: "recordProgress" },
+        done: {
+          guard: "hasNoTerminal",
+          actions: ["recordDone", "rememberDone"],
+        },
+        error: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        abort: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        fail: {
+          guard: "hasNoTerminal",
+          actions: ["recordError", "rememberError"],
+        },
+        traceLimit: {
+          actions: ["recordTraceLimit", "rememberTraceLimit"],
+        },
+      },
+    },
+    selecting: {
+      always: [
+        { guard: "hasDoneTerminal", target: "succeeded" },
+        { guard: "hasErrorTerminal", target: "failed" },
+        {
+          guard: "hasQueuedInterrupt",
+          target: "answering",
+          actions: "takeNextInterrupt",
+        },
+        { target: "waiting" },
+      ],
+    },
+    succeeded: { type: "final", entry: "releaseInterrupts" },
+    failed: { type: "final", entry: "releaseInterrupts" },
+  },
+  output: ({ context }) =>
+    context.terminal ?? {
+      kind: "error",
+      error: { exitReason: "error", reason: "driver session ended without a terminal event" },
+    },
+});
+
+const messageOf = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const interactionError = (
+  phase: "answerer" | "driver-transport",
+  reason: string,
+): DriverSessionOutcome => ({
+  kind: "error",
+  error: { exitReason: "error", reason },
+  operationalFailure: { phase, reason },
+});
+
+export const createDriverSession = (options: {
+  driver: AgentDriver;
+  answerer: Answerer;
+  context: AnswererContext;
+  runDir: OpenedRunDir;
+  interruptCounts: { ask: number; approve: number; custom: number };
+  history: AnswerHistoryEntry[];
+  abortSignal?: AbortSignal;
+  onToolCall: () => void;
+}): DriverSession => {
+  const { driver, answerer, context, runDir, interruptCounts, history, abortSignal, onToolCall } =
+    options;
+  const recordedInteractionFailure = (
+    phase: "answerer" | "driver-transport",
+    reason: string,
+  ): InteractionFailure => {
+    const outcome = interactionError(phase, reason);
+    const failure = runDir.events.append({
+      type: "error",
+      exitReason: "error",
+      reason,
+      ts: Date.now(),
+    });
+    return failure
+      ? new InteractionFailure(traceLimitOutcome({ type: "traceLimit", failure }), true)
+      : new InteractionFailure(outcome, true);
+  };
+  const rejectedTraceEvent = (failure: RunTraceLimitFailure): InteractionFailure =>
+    new InteractionFailure(traceLimitOutcome({ type: "traceLimit", failure }), true);
+  const effects: DriverSessionEffects = {
+    answer: async (interrupt) => {
+      const failure = runDir.events.append({ type: "interrupt", ...interrupt, ts: Date.now() });
+      if (failure) throw rejectedTraceEvent(failure);
+      interruptCounts[interrupt.kind]++;
+      const startedAt = Date.now();
+      try {
+        const answer = AnswerSchema.parse(await answerer.answer(interrupt, context));
+        if (answer.kind !== interrupt.kind || answer.id !== interrupt.id) {
+          throw new Error(
+            `answer correlation mismatch: expected ${interrupt.kind} interrupt '${interrupt.id}', received ${answer.kind} answer '${answer.id}'`,
+          );
+        }
+        return { answer, durationMs: Date.now() - startedAt };
+      } catch (error) {
+        const reason = `answerer failed: ${messageOf(error)}`;
+        throw recordedInteractionFailure("answerer", reason);
+      }
+    },
+    respond: async (interrupt, completed) => {
+      const failure = runDir.events.append({
+        type: "responded",
+        ...completed.answer,
+        ts: Date.now(),
+      });
+      if (failure) throw rejectedTraceEvent(failure);
+      history.push({ interrupt, ...completed });
+      try {
+        await driver.respond(completed.answer);
+      } catch (error) {
+        const reason = `driver respond failed: ${messageOf(error)}`;
+        throw recordedInteractionFailure("driver-transport", reason);
+      }
+    },
+    progress: (progress) => {
+      if (runDir.events.append({ type: "progress", ...progress })) return;
+      if (progress.kind === "tool-call") onToolCall();
+    },
+    terminal: (terminalOutcome, alreadyRecorded) => {
+      if (alreadyRecorded) return;
+      if (terminalOutcome.kind === "done") {
+        runDir.events.append({ type: "done", ...terminalOutcome.done, ts: Date.now() });
+      } else {
+        runDir.events.append({ type: "error", ...terminalOutcome.error, ts: Date.now() });
+      }
+    },
+  };
+
+  const actor = createActor(driverSessionMachine, { input: { effects } });
+  const onInterrupt = (interrupt: Interrupt): Promise<void> => {
+    if (actor.getSnapshot().status === "done") return Promise.resolve();
+    return new Promise((complete) => {
+      actor.send({ type: "interrupt", pending: { interrupt, complete } });
+    });
+  };
+  const onProgress = (progress: Progress): void => {
+    if (actor.getSnapshot().status === "active") actor.send({ type: "progress", progress });
+  };
+  const sendRecordedTerminal = (event: RunEvent, terminalEvent: RecordableTerminalEvent): void => {
+    if (actor.getSnapshot().status !== "active") return;
+    if (runDir.events.append(event)) return;
+    actor.send({ ...terminalEvent, recorded: true });
+  };
+  const onDone = (done: Done): void => {
+    sendRecordedTerminal({ type: "done", ...done, ts: Date.now() }, { type: "done", done });
+  };
+  const onError = (error: ErrorEvent): void => {
+    sendRecordedTerminal({ type: "error", ...error, ts: Date.now() }, { type: "error", error });
+  };
+  const onAbort = (): void => {
+    sendRecordedTerminal(
+      { type: "error", exitReason: "error", reason: "run aborted", ts: Date.now() },
+      { type: "abort" },
+    );
+  };
+  const onTraceLimit = (failure: RunTraceLimitFailure): void => {
+    if (actor.getSnapshot().status === "active") actor.send({ type: "traceLimit", failure });
+  };
+  let stopListeningForTraceLimit = (): void => {};
+  const unbind = (): void => {
+    driver.off("interrupt", onInterrupt);
+    driver.off("progress", onProgress);
+    driver.off("done", onDone);
+    driver.off("error", onError);
+    abortSignal?.removeEventListener("abort", onAbort);
+    stopListeningForTraceLimit();
+  };
+
+  driver.on("interrupt", onInterrupt);
+  driver.on("progress", onProgress);
+  driver.on("done", onDone);
+  driver.on("error", onError);
+  if (!abortSignal?.aborted) abortSignal?.addEventListener("abort", onAbort, { once: true });
+
+  actor.start();
+  stopListeningForTraceLimit = runDir.events.onLimitExceeded(onTraceLimit);
+  if (abortSignal?.aborted) onAbort();
+  const outcome = toPromise(actor).finally(unbind);
+  return {
+    outcome,
+    fail: (reason) => {
+      sendRecordedTerminal(
+        { type: "error", exitReason: "error", reason, ts: Date.now() },
+        { type: "fail", reason },
+      );
+    },
+  };
+};
+
+// Internal export for transition-level tests.
+export { driverSessionMachine };

package/src/orchestrator/evaluator-identity.ts

@@ -0,0 +1,45 @@
+import { readFile } from "node:fs/promises";
+import { dirname, extname, join, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { contentHash, valueHash } from "../core/index.js";
+import { walkAuthoredFiles } from "./fs-read.js";
+
+const currentDistributionRoot = (): string => {
+  const modulePath = fileURLToPath(import.meta.url);
+  const parent = dirname(dirname(modulePath));
+  const leaf = parent.split(sep).at(-1);
+  if (leaf !== "src" && leaf !== "lib") {
+    throw new Error(`cannot locate Lux evaluator distribution from ${modulePath}`);
+  }
+  return parent;
+};
+
+const isRuntimeImplementation = (path: string): boolean => {
+  if (path.endsWith(".test.ts") || path.endsWith(".test.js")) return false;
+  if (path.endsWith("test-helpers.ts") || path.endsWith("test-helpers.js")) return false;
+  if (path.split(sep).includes("fixtures")) return false;
+  const extension = extname(path);
+  return extension === ".ts" || extension === ".js" || extension === ".json";
+};
+
+/**
+ * Hash the exact Lux source/build tree executing the evaluation. Package
+ * versions alone are insufficient in development, where evaluator semantics
+ * can change without a release bump.
+ */
+export const evaluatorImplementationHash = async (
+  distributionRoot: string = currentDistributionRoot(),
+): Promise<string> => {
+  const root = resolve(distributionRoot);
+  const files = (await walkAuthoredFiles(root)).filter(isRuntimeImplementation);
+  if (files.length === 0) {
+    throw new Error(`Lux evaluator distribution has no implementation files: ${root}`);
+  }
+  const evidence = await Promise.all(
+    files.map(async (path) => ({
+      path: relative(root, join(root, path)).split(sep).join("/"),
+      hash: contentHash(await readFile(join(root, path))),
+    })),
+  );
+  return valueHash(evidence);
+};

package/src/orchestrator/experiment-budget.ts

@@ -0,0 +1,61 @@
+import { type ExperimentCampaign, totalTokens } from "../core/index.js";
+import type { CampaignExecutionOptions } from "./experiment-context.js";
+import { experimentActiveElapsedMs, type OpenedExperimentRun } from "./experiment-runs.js";
+
+export const throwIfAborted = (signal: AbortSignal | undefined): void => {
+  if (!signal?.aborted) return;
+  throw signal.reason instanceof Error ? signal.reason : new Error("experiment aborted");
+};
+
+export const boundedExecutionSignal = (
+  execution: CampaignExecutionOptions,
+  experiment: OpenedExperimentRun,
+): AbortSignal => {
+  const elapsed = experimentActiveElapsedMs(experiment.manifest);
+  const remaining = execution.loaded.campaign.budget.maxWallTimeMs - elapsed;
+  if (remaining <= 0) {
+    throw new Error(
+      `experiment exceeded its wall-time budget ${execution.loaded.campaign.budget.maxWallTimeMs}ms`,
+    );
+  }
+  const deadline = AbortSignal.timeout(Math.max(1, remaining));
+  return execution.abortSignal ? AbortSignal.any([execution.abortSignal, deadline]) : deadline;
+};
+
+export const ensureBudget = (
+  campaign: ExperimentCampaign,
+  experiment: OpenedExperimentRun,
+  additional: { metricCalls?: number; judgeCalls?: number; proposalCalls?: number } = {},
+): void => {
+  const usage = experiment.manifest.budgetUsage;
+  const elapsed = experimentActiveElapsedMs(experiment.manifest);
+  const checks: [boolean, string][] = [
+    [
+      usage.metricCalls + (additional.metricCalls ?? 0) > campaign.budget.maxMetricCalls,
+      `metric-call budget ${campaign.budget.maxMetricCalls}`,
+    ],
+    [
+      usage.judgeCalls + (additional.judgeCalls ?? 0) > campaign.budget.maxJudgeCalls,
+      `judge-call budget ${campaign.budget.maxJudgeCalls}`,
+    ],
+    [
+      usage.proposalCalls + (additional.proposalCalls ?? 0) > campaign.budget.maxProposalCalls,
+      `proposal-call budget ${campaign.budget.maxProposalCalls}`,
+    ],
+    [
+      totalTokens(usage.usage) > campaign.budget.maxTokens,
+      `token budget ${campaign.budget.maxTokens}`,
+    ],
+    [usage.costUsd > campaign.budget.maxCostUsd, `cost budget $${campaign.budget.maxCostUsd}`],
+    [
+      (usage.unpricedModelCalls ?? 0) > campaign.budget.maxUnpricedModelCalls,
+      `unpriced-model-call budget ${campaign.budget.maxUnpricedModelCalls}`,
+    ],
+    [
+      elapsed > campaign.budget.maxWallTimeMs,
+      `wall-time budget ${campaign.budget.maxWallTimeMs}ms`,
+    ],
+  ];
+  const exceeded = checks.find(([failed]) => failed);
+  if (exceeded) throw new Error(`experiment exceeded its ${exceeded[1]}`);
+};

package/src/orchestrator/experiment-context.ts

@@ -0,0 +1,41 @@
+import type { Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+import type { discoverCases } from "./discover.js";
+import type { LoadedExperimentCampaign } from "./experiment-loader.js";
+
+export type EvaluationCorpus =
+  | {
+      kind: "agent";
+      cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
+      hash: string;
+    }
+  | {
+      kind: "grader";
+      hash: string;
+    };
+
+export type CampaignExecutionOptions = {
+  config: ResolvedLuxConfig;
+  registry: Registry;
+  loaded: LoadedExperimentCampaign;
+  runsRoot?: string;
+  resumeFrom?: string;
+  abortSignal?: AbortSignal;
+  onEvaluationUnit?: (unit: {
+    unitId: string;
+    candidateId: string;
+    profileId: string;
+    split: "train" | "validation" | "test";
+  }) => void | Promise<void>;
+};
+
+export const profileOrThrow = (config: ResolvedLuxConfig, profileId: string) => {
+  const profile = config.profiles?.[profileId];
+  if (!profile) {
+    const available = Object.keys(config.profiles ?? {}).sort();
+    throw new Error(
+      `experiment profile '${profileId}' is not configured${available.length > 0 ? `; available: ${available.join(", ")}` : ""}`,
+    );
+  }
+  return profile;
+};

package/src/orchestrator/experiment-contracts.ts

@@ -0,0 +1,43 @@
+import { z } from "zod";
+import { CandidateSchema, ExperimentBudgetSchema } from "../core/index.js";
+import { ExperimentRunSchema } from "./experiment-runs.js";
+
+export const CampaignExecutionResultSchema = z
+  .object({
+    experimentDir: z.string(),
+    champion: CandidateSchema.nullable(),
+    manifest: ExperimentRunSchema,
+  })
+  .strict();
+export type CampaignExecutionResult = z.infer<typeof CampaignExecutionResultSchema>;
+
+export const ExperimentCampaignPlanSchema = z
+  .object({
+    campaignId: z.string(),
+    mode: z.enum(["experiment", "optimize"]),
+    evaluationKind: z.enum(["agent", "grader"]),
+    subjectRoot: z.string(),
+    mutableComponents: z.array(z.string()),
+    profiles: z.array(z.string()),
+    splits: z.array(
+      z
+        .object({
+          split: z.enum(["train", "validation", "test"]),
+          examples: z.number().int().nonnegative(),
+        })
+        .strict(),
+    ),
+    projected: z
+      .object({
+        candidates: z.number().int().nonnegative(),
+        metricCalls: z.number().int().nonnegative(),
+        judgeCalls: z.number().int().nonnegative(),
+        proposalCalls: z.number().int().nonnegative(),
+      })
+      .strict(),
+    budget: ExperimentBudgetSchema,
+    fitsProjectedCallBudgets: z.boolean(),
+    warnings: z.array(z.string()),
+  })
+  .strict();
+export type ExperimentCampaignPlan = z.infer<typeof ExperimentCampaignPlanSchema>;