@prismatic-io/lux 0.0.2-preview.17 → 0.0.2-preview.18

package/src/orchestrator/experiment-corpus.ts

@@ -13,7 +13,11 @@ import {
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
 import { experimentDriverIdentity } from "./driver-identity.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import type {
+  CampaignExecutionOptions,
+  CorpusSelection,
+  EvaluationCorpus,
+} from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
@@ -252,6 +256,21 @@ export const graderCorpusHash = async (
 export const prepareCorpus = async (
   execution: CampaignExecutionOptions,
 ): Promise<EvaluationCorpus> => {
+  const corpus = await inspectCorpus(execution);
+  if (corpus.kind === "grader") return corpus;
+  return {
+    ...corpus,
+    hash: await agentCorpusHash(
+      execution,
+      corpus.cases,
+      selectorsBySplit(execution.loaded.campaign),
+    ),
+  };
+};
+
+export const inspectCorpus = async (
+  execution: CampaignExecutionOptions,
+): Promise<CorpusSelection> => {
   const campaign = execution.loaded.campaign;
   const splits = selectorsBySplit(campaign);
   if (campaign.evaluation.kind === "grader") {
@@ -344,7 +363,6 @@ export const prepareCorpus = async (
   return {
     kind: "agent",
     cases: discovery.cases,
-    hash: await agentCorpusHash(execution, discovery.cases, splits),
   };
 };
 

package/src/orchestrator/experiment-identity.ts

@@ -253,9 +253,9 @@ const authoredConfigDependencyHash = async (
   }
   return authoredDependencyGraphHash(configRoot, [...entrypoints]);
 };
-const validateAgentCaseProfileDrivers = async (
+export const validateAgentCaseProfileDrivers = async (
   execution: CampaignExecutionOptions,
-  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+  corpus: Pick<Extract<EvaluationCorpus, { kind: "agent" }>, "cases">,
 ): Promise<void> => {
   const selected = [
     ...new Map(

package/src/orchestrator/experiment-preflight.ts

@@ -0,0 +1,27 @@
+import { assertValidationArgumentsBound } from "./candidates.js";
+import {
+  type CampaignExecutionOptions,
+  type CorpusSelection,
+  profileOrThrow,
+} from "./experiment-context.js";
+import { inspectCorpus, prepareCorpus } from "./experiment-corpus.js";
+import { evaluationIdentity, validateAgentCaseProfileDrivers } from "./experiment-identity.js";
+
+export const preparePlanningCorpus = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+  staticChecks: boolean,
+): Promise<CorpusSelection> => {
+  if (!staticChecks) {
+    const corpus = await prepareCorpus(execution);
+    await evaluationIdentity(execution, corpus, mode);
+    return corpus;
+  }
+  const corpus = await inspectCorpus(execution);
+  const campaign = execution.loaded.campaign;
+  for (const profileId of campaign.profiles) profileOrThrow(execution.config, profileId);
+  if (corpus.kind === "agent") await validateAgentCaseProfileDrivers(execution, corpus);
+  for (const validation of campaign.subject.validate)
+    assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+  return corpus;
+};

package/src/orchestrator/experiment.ts

@@ -48,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -182,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -192,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({