@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/experiment-corpus.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import {
   bindSubjectRoot,
   type Candidate,
@@ -11,16 +12,14 @@ import {
 } from "../core/index.js";
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import { experimentDriverIdentity } from "./driver-identity.js";
+import type {
+  CampaignExecutionOptions,
+  CorpusSelection,
+  EvaluationCorpus,
+} from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  referencedAnswererCli,
-  referencedCli,
-} from "./experiment-runtime-identity.js";
+import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
 import { loadEvalCase } from "./loader.js";
 import { resolveAnswererReference, resolveDriverReference } from "./orchestrator.js";
@@ -123,8 +122,6 @@ export const agentCorpusHash = async (
         .map((evalCase) => [evalCase.path, evalCase]),
     ).values(),
   ];
-  const caseBehaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
   const definitions = await Promise.all(
     selected.map(async (evalCase) => {
       const authored = await loadEvalCase(evalCase.path);
@@ -133,39 +130,31 @@ export const agentCorpusHash = async (
         execution.loaded.campaign.profiles.map(async (profileId) => {
           const profile = profileOrThrow(execution.config, profileId);
           const harness = profile.harness ?? execution.config.harness;
-          const driver = resolveDriverReference(
-            bound,
-            profile.driver ?? execution.config.defaultDriver,
+          const driver = bindSubjectRoot(
+            resolveDriverReference(
+              bound,
+              profile.driver ?? execution.config.defaultDriver,
+              undefined,
+              execution.config.driverDefaults,
+            ),
+            execution.loaded.subjectRoot,
           );
-          const answerer = resolveAnswererReference(
-            bound,
-            profile.answerer ?? execution.config.defaultAnswerer,
-            harness,
+          const answerer = bindSubjectRoot(
+            resolveAnswererReference(
+              bound,
+              profile.answerer ?? execution.config.defaultAnswerer,
+              harness,
+            ),
+            execution.loaded.subjectRoot,
           );
-          const driverCli = referencedCli(driver);
           const answererCli = referencedAnswererCli(answerer);
-          const configuredRoot =
-            driverCli?.name === "claude-code" ? claudeConfigRoot(driver) : null;
-          let driverClaudeBehaviorHash: string | null = null;
-          if (configuredRoot) {
-            let hash = caseBehaviorHashes.get(configuredRoot);
-            if (!hash) {
-              hash = claudeUserBehaviorHash(configuredRoot);
-              caseBehaviorHashes.set(configuredRoot, hash);
-            }
-            driverClaudeBehaviorHash = await hash;
-          }
           return {
             profileId,
             driver,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            driverClaudeBehaviorHash,
-            configuredDriverSources: await configuredClaudeSourceHashes(
+            driverIdentity: await experimentDriverIdentity(
               driver,
+              execution.registry,
               execution.config.rootDir,
-              configuredSourceHashes,
             ),
             answerer,
             answererCliIdentity: answererCli
@@ -275,6 +264,21 @@ export const graderCorpusHash = async (
 export const prepareCorpus = async (
   execution: CampaignExecutionOptions,
 ): Promise<EvaluationCorpus> => {
+  const corpus = await inspectCorpus(execution);
+  if (corpus.kind === "grader") return corpus;
+  return {
+    ...corpus,
+    hash: await agentCorpusHash(
+      execution,
+      corpus.cases,
+      selectorsBySplit(execution.loaded.campaign),
+    ),
+  };
+};
+
+export const inspectCorpus = async (
+  execution: CampaignExecutionOptions,
+): Promise<CorpusSelection> => {
   const campaign = execution.loaded.campaign;
   const splits = selectorsBySplit(campaign);
   if (campaign.evaluation.kind === "grader") {
@@ -303,6 +307,24 @@ export const prepareCorpus = async (
     );
   }
   const owners = new Map<string, string>();
+  assertDisjointClusters(
+    splits.map((split) => ({
+      name: split.name,
+      items: selectCases(discovery.cases, split.selector),
+    })),
+  );
+  for (const constraint of campaign.objectives.constraints) {
+    if (constraint.confidence?.method !== "wilson") continue;
+    const selector = splits.find((split) => split.name === constraint.split)?.selector;
+    if (
+      selector &&
+      selectCases(discovery.cases, selector).some((evalCase) => evalCase.cluster !== undefined)
+    ) {
+      throw new Error(
+        "authored case clusters produce bounded means; use Hoeffding or percentile-bootstrap confidence instead of Wilson",
+      );
+    }
+  }
   const caseIds = new Map<string, string>();
   const evidenceOwners = new Map<
     string,
@@ -349,7 +371,6 @@ export const prepareCorpus = async (
   return {
     kind: "agent",
     cases: discovery.cases,
-    hash: await agentCorpusHash(execution, discovery.cases, splits),
   };
 };
 

package/src/orchestrator/experiment-effects.ts

@@ -0,0 +1,63 @@
+import { z } from "zod";
+import { ConfidenceConfigSchema } from "../core/index.js";
+import type { ExperimentRun } from "./experiment-runs.js";
+import { candidateEvaluations } from "./experiment-selection.js";
+import { PairedEffectSchema, pairedEffect } from "./paired-effects.js";
+
+export const ExperimentEffectsSchema = z.array(
+  PairedEffectSchema.extend({
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    metric: z.string(),
+  }),
+);
+
+export const experimentEffects = (
+  experiment: ExperimentRun,
+): z.infer<typeof ExperimentEffectsSchema> => {
+  const candidateIds =
+    experiment.selection?.decisionCandidateIds ??
+    experiment.candidates.map((candidate) => candidate.id);
+  const baseline = candidateEvaluations(experiment, "baseline");
+  const metrics = new Set([
+    "quality.passRate",
+    experiment.evaluationKind === "grader" ? "harness.total" : "agent.total",
+    ...(experiment.evaluationKind === "agent" ? ["score", "interrupts.ask", "toolCalls"] : []),
+    ...experiment.campaignSnapshot.objectives.constraints
+      .filter((constraint) => constraint.relativeTo)
+      .map((constraint) => constraint.metric),
+  ]);
+  return candidateIds
+    .filter((id) => id !== "baseline")
+    .flatMap((candidateId) =>
+      candidateEvaluations(experiment, candidateId).flatMap((after) =>
+        [...metrics].map((metric) => {
+          const before = baseline.find(
+            (entry) => entry.profileId === after.profileId && entry.split === after.split,
+          );
+          const constraint = experiment.campaignSnapshot.objectives.constraints.find(
+            (entry) =>
+              entry.relativeTo &&
+              entry.metric === metric &&
+              entry.split === after.split &&
+              (!entry.profile || entry.profile === after.profileId),
+          );
+          const confidence =
+            constraint?.confidence ??
+            ConfidenceConfigSchema.parse({
+              method: ["quality.passRate", "score"].includes(metric)
+                ? "hoeffding"
+                : "percentile-bootstrap",
+            });
+          return {
+            candidateId,
+            profileId: after.profileId,
+            split: after.split,
+            metric,
+            ...pairedEffect(before, after, metric, confidence),
+          };
+        }),
+      ),
+    );
+};

package/src/orchestrator/experiment-evaluation.ts

@@ -2,10 +2,7 @@ import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   addUsage,
   type Candidate,
@@ -17,6 +14,7 @@ import {
   valueHash,
 } from "../core/index.js";
 import { materializeCandidate } from "./candidates.js";
+import { runComparisonIdentity } from "./comparison-identity.js";
 import { readContentBoundJson, writeContentBoundJson } from "./content-bound-json.js";
 import { type discoverCases, selectCases } from "./discover.js";
 import { ensureBudget, throwIfAborted } from "./experiment-budget.js";
@@ -25,8 +23,10 @@ import { profileOrThrow } from "./experiment-context.js";
 import type { ExperimentEvaluation, OpenedExperimentRun } from "./experiment-runs.js";
 import { average } from "./experiment-selection.js";
 import { evaluateGraderCandidate } from "./grader-experiment.js";
+import { registryWithHarness } from "./harness-registry.js";
 import { loadEvalCase } from "./loader.js";
 import { Orchestrator, type SuiteCaseResult, type SuiteItem } from "./orchestrator.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadSuiteSummary, type SuiteSummary } from "./suite-summary.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
@@ -44,6 +44,7 @@ const gradingFailureMessage = (results: readonly SuiteCaseResult[]): string | un
     }
     const report = result.result?.grading;
     const failedAssertion = report.assertions.find((assertion) => {
+      if (assertion.passed === null) return false;
       if (assertion.errorMessage) return true;
       const detail =
         assertion.detail && typeof assertion.detail === "object"
@@ -101,6 +102,23 @@ const evaluationMetrics = (
   ),
 });
 
+const evaluationMetricDefinitions = (summary: SuiteSummary) =>
+  Object.fromEntries(
+    Object.entries(summary.metrics).flatMap(([name, metric]) => {
+      if (!metric.unit && !metric.label && !metric.better) return [];
+      return [
+        [
+          name,
+          {
+            unit: metric.unit ?? "number",
+            better: metric.better ?? "neutral",
+            ...(metric.label ? { label: metric.label } : {}),
+          },
+        ],
+      ];
+    }),
+  );
+
 const summaryUsage = (summary: SuiteSummary) =>
   summary.samples.reduce(
     (usage, sample) =>
@@ -263,8 +281,38 @@ const agentEvaluationFromSuite = (input: {
   summary: SuiteSummary;
   results: readonly SuiteCaseResult[];
   suiteDir: string;
+  evaluationHash: string | undefined;
+  clusterId: string | undefined;
 }): CandidateEvaluation => {
   const { candidate, profileId, split, summary, results, suiteDir } = input;
+  const caseSamples = clusterCaseSamples(
+    summary.samples.map((sample) => {
+      const metadata = results.find((result) => result.caseId === sample.caseId)?.result?.metadata;
+      // Validated assertion configs contain a fresh candidate workspace path.
+      // The campaign identity binds the authored evaluator without that incidental path.
+      const identityHash =
+        metadata?.provenance && input.evaluationHash
+          ? runComparisonIdentity({
+              ...metadata,
+              provenance: { ...metadata.provenance, evaluatorConfigHash: input.evaluationHash },
+            })
+          : null;
+      return {
+        [sample.canonicalCaseId]: {
+          identityHash,
+          scoreBasis: sample.scoreBasis ?? null,
+          ...(input.clusterId ? { clusterId: input.clusterId } : {}),
+          metrics: Object.fromEntries(
+            Object.entries({
+              ...sample.metrics,
+              ...(sample.score !== undefined ? { score: sample.score } : {}),
+              "quality.passRate": sample.passed === true ? 1 : 0,
+            }).map(([name, value]) => [name, { mean: value, n: 1 }]),
+          ),
+        },
+      };
+    }),
+  );
   const modelAccounting = results.map((result) =>
     modelAccountingFromMetadata(result.result?.metadata ?? null),
   );
@@ -277,7 +325,12 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
+    metricDefinitions: {
+      score: { unit: "score", better: "higher", label: "Quality score" },
+      ...evaluationMetricDefinitions(summary),
+    },
     samples: evaluationSamples(summary),
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
       (total, result) =>
@@ -344,7 +397,7 @@ const aggregateAgentUnits = (
       caseValues.set(evaluatedCaseId, values);
     }
   }
-  const samples = Object.fromEntries(
+  const ungroupedSamples = Object.fromEntries(
     [...samplesByCase].map(([name, byCase]) => [
       name,
       [...byCase.entries()]
@@ -352,6 +405,19 @@ const aggregateAgentUnits = (
         .map(([, values]) => average(values)),
     ]),
   );
+  const metricDefinitions = Object.assign(
+    {},
+    ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
+  );
+  const caseSamples = clusterCaseSamples(units.map((unit) => unit.evaluation.caseSamples));
+  const grouped = groupedCaseMetricSamples(caseSamples, repetitions);
+  const samples = grouped ?? ungroupedSamples;
+  const completeScore =
+    caseSamples &&
+    Object.values(caseSamples).every(
+      (sample) => sample.metrics.score?.n === repetitions && Boolean(sample.scoreBasis),
+    );
+  if (!completeScore) delete samples.score;
   return {
     candidateId: candidate.id,
     profileId,
@@ -362,12 +428,20 @@ const aggregateAgentUnits = (
       [...caseValues].map(([caseId, values]) => [caseId, average(values)]),
     ),
     metrics: Object.fromEntries(
-      [...metricValues].map(([name, values]) => [
-        name,
-        name === "subject.bytes" ? candidate.sourceBytes : average(values),
-      ]),
+      [...metricValues]
+        .filter(([name]) =>
+          name === "score"
+            ? completeScore
+            : !grouped || name === "subject.bytes" || Object.hasOwn(grouped, name),
+        )
+        .map(([name, values]) => [
+          name,
+          name === "subject.bytes" ? candidate.sourceBytes : average(grouped?.[name] ?? values),
+        ]),
     ),
+    metricDefinitions,
     samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
     unpricedModelCalls: units.reduce(
@@ -436,8 +510,10 @@ export const evaluateAgentCandidate = async (options: {
   const profile = profileOrThrow(execution.config, profileId);
   const harness = profile.harness ?? execution.config.harness;
   const defaultDriver = profile.driver ?? execution.config.defaultDriver;
-  const registry = execution.registry.withAssertion(createRubricAssertion(harness));
+  const registry = registryWithHarness(execution.registry, execution.config, harness);
   const orchestrator = new Orchestrator({
+    driverDefaults: execution.config.driverDefaults,
+    environment: profile.environment ?? execution.config.environment,
     registry,
     defaultAnswerer: profile.answerer ?? execution.config.defaultAnswerer,
     ...(defaultDriver ? { defaultDriver } : {}),
@@ -606,6 +682,8 @@ export const evaluateAgentCandidate = async (options: {
             summary,
             results: suite.results,
             suiteDir: suite.suiteDir,
+            evaluationHash: experiment.manifest.evaluationHash,
+            clusterId: evalCase.cluster,
           });
         } finally {
           await rm(unitRoot, { recursive: true, force: true });

package/src/orchestrator/experiment-identity.ts

@@ -5,20 +5,17 @@ import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
 import { authoredDependencyGraphHash } from "./authored-dependencies.js";
 import { assertValidationArgumentsBound } from "./candidates.js";
 import { selectCases } from "./discover.js";
+import { experimentDriverIdentity, referencedDriverRuntime } from "./driver-identity.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { caseRubricCliReferences, selectorsBySplit } from "./experiment-corpus.js";
 import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
   cliIdentity,
-  configuredClaudeSourceHashes,
   LUX_VERSION,
   type RuntimeCliReference,
   RuntimeCliReferenceSchema,
   referencedAnswererCli,
-  referencedCli,
   runtimeCliIdentityHashForReferences,
 } from "./experiment-runtime-identity.js";
 import { loadEvalCase } from "./loader.js";
@@ -33,15 +30,6 @@ import { validatePluginConfig } from "./validate-plugin-config.js";
 export const resolvedProfileSnapshot = async (
   execution: CampaignExecutionOptions,
 ): Promise<Record<string, unknown>> => {
-  const behaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
-  const behaviorHash = (root: string): Promise<string | null> => {
-    const existing = behaviorHashes.get(root);
-    if (existing) return existing;
-    const created = claudeUserBehaviorHash(root);
-    behaviorHashes.set(root, created);
-    return created;
-  };
   return Object.fromEntries(
     await Promise.all(
       execution.loaded.campaign.profiles.map(async (profileId) => {
@@ -53,20 +41,12 @@ export const resolvedProfileSnapshot = async (
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.driver ?? execution.config.defaultDriver)
             : undefined;
-        const driverCli = referencedCli(driver);
         const harness = profile.harness ?? execution.config.harness;
         const answerer =
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.answerer ?? execution.config.defaultAnswerer)
             : undefined;
         const answererCli = referencedAnswererCli(answerer);
-        const driverUserBehaviorHash =
-          driverCli?.name === "claude-code" ? await behaviorHash(claudeConfigRoot(driver)) : null;
-        const driverConfiguredSources = await configuredClaudeSourceHashes(
-          driver,
-          execution.config.rootDir,
-          configuredSourceHashes,
-        );
         let harnessCommand = harness?.command;
         if (!harnessCommand) {
           harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
@@ -80,17 +60,15 @@ export const resolvedProfileSnapshot = async (
           profileId,
           {
             id: profileId,
+            environment: profile.environment ?? execution.config.environment ?? null,
+            configuredConcurrency: execution.config.concurrency,
+            evaluationConcurrency: 1,
             driver: driver ?? null,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            ...(driverCli?.name === "claude-code"
-              ? {
-                  driverClaudeConfigRoot: claudeConfigRoot(driver),
-                  driverUserBehaviorHash,
-                  driverConfiguredSources,
-                }
-              : {}),
+            driverIdentity: await experimentDriverIdentity(
+              driver,
+              execution.registry,
+              execution.config.rootDir,
+            ),
             answerer,
             answererCliIdentity: answererCli
               ? await cliIdentity(answererCli.command, execution.config.rootDir)
@@ -130,7 +108,11 @@ const trustedExecutableEntrypoints = async (
   }
   for (const profileId of execution.loaded.campaign.profiles) {
     const profile = profileOrThrow(execution.config, profileId);
-    add(referencedCli(profile.driver ?? execution.config.defaultDriver)?.command);
+    for (const command of referencedDriverRuntime(
+      profile.driver ?? execution.config.defaultDriver,
+      execution.registry,
+    ).commands)
+      add(command);
     add(referencedAnswererCli(profile.answerer ?? execution.config.defaultAnswerer)?.command);
     add((profile.harness ?? execution.config.harness)?.command);
   }
@@ -147,7 +129,20 @@ const trustedExecutableEntrypoints = async (
         await loadEvalCase(evalCase.path),
         execution.loaded.subjectRoot,
       );
-      add(referencedCli(authored.driver)?.command);
+      for (const profileId of execution.loaded.campaign.profiles) {
+        const profile = profileOrThrow(execution.config, profileId);
+        const driver = bindSubjectRoot(
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
+          execution.loaded.subjectRoot,
+        );
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command);
+      }
       add(referencedAnswererCli(authored.answerer)?.command);
       for (const reference of caseRubricCliReferences(execution, authored)) {
         add(reference.command);
@@ -207,16 +202,25 @@ export const runtimeCliReferences = async (
       for (const profileId of execution.loaded.campaign.profiles) {
         const profile = profileOrThrow(execution.config, profileId);
         const harness = profile.harness ?? execution.config.harness;
-        const driver = resolveDriverReference(
-          authored,
-          profile.driver ?? execution.config.defaultDriver,
+        const driver = bindSubjectRoot(
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
+          execution.loaded.subjectRoot,
         );
-        const answerer = resolveAnswererReference(
-          authored,
-          profile.answerer ?? execution.config.defaultAnswerer,
-          harness,
+        const answerer = bindSubjectRoot(
+          resolveAnswererReference(
+            authored,
+            profile.answerer ?? execution.config.defaultAnswerer,
+            harness,
+          ),
+          execution.loaded.subjectRoot,
         );
-        add(referencedCli(driver)?.command, execution.config.rootDir);
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command, execution.config.rootDir);
         add(referencedAnswererCli(answerer)?.command, execution.config.rootDir);
       }
       for (const reference of caseRubricCliReferences(execution, authored)) {
@@ -262,9 +266,9 @@ const authoredConfigDependencyHash = async (
   }
   return authoredDependencyGraphHash(configRoot, [...entrypoints]);
 };
-const validateAgentCaseProfileDrivers = async (
+export const validateAgentCaseProfileDrivers = async (
   execution: CampaignExecutionOptions,
-  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+  corpus: Pick<Extract<EvaluationCorpus, { kind: "agent" }>, "cases">,
 ): Promise<void> => {
   const selected = [
     ...new Map(
@@ -280,13 +284,17 @@ const validateAgentCaseProfileDrivers = async (
     );
     for (const profileId of execution.loaded.campaign.profiles) {
       const profile = profileOrThrow(execution.config, profileId);
-      const reference = resolveDriverReference(
-        authored,
-        profile.driver ?? execution.config.defaultDriver,
+      const reference = bindSubjectRoot(
+        resolveDriverReference(
+          authored,
+          profile.driver ?? execution.config.defaultDriver,
+          undefined,
+          execution.config.driverDefaults,
+        ),
+        execution.loaded.subjectRoot,
       );
       const name = typeof reference === "string" ? reference : reference.name;
       const rawConfig = typeof reference === "object" ? reference.config : undefined;
-      boundExperimentDriverConfig(name, rawConfig, "identity-check");
       const plugin = execution.registry.getDriver(name);
       if (!plugin) {
         throw new Error(
@@ -298,7 +306,7 @@ const validateAgentCaseProfileDrivers = async (
         rawConfig,
         `case '${evalCase.id}' profile '${profileId}' driver '${name}'`,
       );
-      boundExperimentDriverConfig(name, validated, "identity-check");
+      boundExperimentDriverConfig(plugin, validated, "identity-check");
     }
   }
 };
@@ -334,6 +342,7 @@ export const evaluationIdentity = async (
     luxVersion: LUX_VERSION,
     evaluatorImplementationHash: await evaluatorImplementationHash(),
     runtime: runtimeIdentity(),
+    concurrency: execution.config.concurrency,
     corpusHash: corpus.hash,
     profiles,
     configSourceHash,

package/src/orchestrator/experiment-preflight.ts

@@ -0,0 +1,27 @@
+import { assertValidationArgumentsBound } from "./candidates.js";
+import {
+  type CampaignExecutionOptions,
+  type CorpusSelection,
+  profileOrThrow,
+} from "./experiment-context.js";
+import { inspectCorpus, prepareCorpus } from "./experiment-corpus.js";
+import { evaluationIdentity, validateAgentCaseProfileDrivers } from "./experiment-identity.js";
+
+export const preparePlanningCorpus = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+  staticChecks: boolean,
+): Promise<CorpusSelection> => {
+  if (!staticChecks) {
+    const corpus = await prepareCorpus(execution);
+    await evaluationIdentity(execution, corpus, mode);
+    return corpus;
+  }
+  const corpus = await inspectCorpus(execution);
+  const campaign = execution.loaded.campaign;
+  for (const profileId of campaign.profiles) profileOrThrow(execution.config, profileId);
+  if (corpus.kind === "agent") await validateAgentCaseProfileDrivers(execution, corpus);
+  for (const validation of campaign.subject.validate)
+    assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+  return corpus;
+};