@prismatic-io/lux 0.0.2-preview.16 → 0.0.2-preview.18

package/src/orchestrator/doctor.ts

@@ -62,6 +62,7 @@ export const DoctorOptionsSchema = z
   .object({
     cwd: z.string().min(1),
     campaigns: z.array(z.string()).optional(),
+    static: z.boolean().optional(),
   })
   .strict();
 export type DoctorOptions = z.input<typeof DoctorOptionsSchema>;
@@ -666,6 +667,7 @@ const inspectCampaign = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   try {
     const loaded = await loadExperimentCampaign(path);
@@ -679,7 +681,9 @@ const inspectCampaign = async (
         placeholders.join(", "),
       );
     }
-    const plan = await planExperimentCampaign({ config, registry, loaded }, mode);
+    const plan = await planExperimentCampaign({ config, registry, loaded }, mode, {
+      static: staticChecks,
+    });
     recorder.campaigns.push({ path, campaignId: loaded.campaign.id, mode, plan });
     recorder.add(
       `campaign:${loaded.campaign.id}`,
@@ -711,6 +715,7 @@ const inspectCampaigns = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   let paths: string[];
   try {
@@ -733,7 +738,7 @@ const inspectCampaigns = async (
     );
   }
   for (const path of paths) {
-    await inspectCampaign(path, config, registry, recorder);
+    await inspectCampaign(path, config, registry, recorder, staticChecks);
   }
 };
 
@@ -749,12 +754,26 @@ export const doctorProject = async (options: DoctorOptions): Promise<DoctorRepor
   validateConfiguredDrivers(config, registry, recorder);
   validateConfiguredAnswerers(config, registry, recorder);
   const caseCliTargets = await inspectCases(config, registry, recorder);
-  await inspectCliTargets(
-    [...configuredCliTargets(config, registry), ...caseCliTargets],
+  if (parsedOptions.static) {
+    recorder.add(
+      "static-checks",
+      "warn",
+      "static checks only; installed executables and campaign execution identity were not checked",
+    );
+  } else {
+    await inspectCliTargets(
+      [...configuredCliTargets(config, registry), ...caseCliTargets],
+      config,
+      recorder,
+    );
+  }
+  await inspectCampaigns(
+    parsedOptions.campaigns ?? [],
     config,
+    registry,
     recorder,
+    parsedOptions.static ?? false,
   );
-  await inspectCampaigns(parsedOptions.campaigns ?? [], config, registry, recorder);
   return recorder.finish(config.rootDir);
 };
 

package/src/orchestrator/experiment-context.ts

@@ -3,17 +3,18 @@ import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
 
-export type EvaluationCorpus =
+export type CorpusSelection =
   | {
       kind: "agent";
       cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
-      hash: string;
     }
   | {
       kind: "grader";
       hash: string;
     };
 
+export type EvaluationCorpus = CorpusSelection & { hash: string };
+
 export type CampaignExecutionOptions = {
   config: ResolvedLuxConfig;
   registry: Registry;

package/src/orchestrator/experiment-corpus.ts

@@ -13,7 +13,11 @@ import {
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
 import { experimentDriverIdentity } from "./driver-identity.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import type {
+  CampaignExecutionOptions,
+  CorpusSelection,
+  EvaluationCorpus,
+} from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
@@ -252,6 +256,21 @@ export const graderCorpusHash = async (
 export const prepareCorpus = async (
   execution: CampaignExecutionOptions,
 ): Promise<EvaluationCorpus> => {
+  const corpus = await inspectCorpus(execution);
+  if (corpus.kind === "grader") return corpus;
+  return {
+    ...corpus,
+    hash: await agentCorpusHash(
+      execution,
+      corpus.cases,
+      selectorsBySplit(execution.loaded.campaign),
+    ),
+  };
+};
+
+export const inspectCorpus = async (
+  execution: CampaignExecutionOptions,
+): Promise<CorpusSelection> => {
   const campaign = execution.loaded.campaign;
   const splits = selectorsBySplit(campaign);
   if (campaign.evaluation.kind === "grader") {
@@ -344,7 +363,6 @@ export const prepareCorpus = async (
   return {
     kind: "agent",
     cases: discovery.cases,
-    hash: await agentCorpusHash(execution, discovery.cases, splits),
   };
 };
 

package/src/orchestrator/experiment-identity.ts

@@ -253,9 +253,9 @@ const authoredConfigDependencyHash = async (
   }
   return authoredDependencyGraphHash(configRoot, [...entrypoints]);
 };
-const validateAgentCaseProfileDrivers = async (
+export const validateAgentCaseProfileDrivers = async (
   execution: CampaignExecutionOptions,
-  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+  corpus: Pick<Extract<EvaluationCorpus, { kind: "agent" }>, "cases">,
 ): Promise<void> => {
   const selected = [
     ...new Map(

package/src/orchestrator/experiment-preflight.ts

@@ -0,0 +1,27 @@
+import { assertValidationArgumentsBound } from "./candidates.js";
+import {
+  type CampaignExecutionOptions,
+  type CorpusSelection,
+  profileOrThrow,
+} from "./experiment-context.js";
+import { inspectCorpus, prepareCorpus } from "./experiment-corpus.js";
+import { evaluationIdentity, validateAgentCaseProfileDrivers } from "./experiment-identity.js";
+
+export const preparePlanningCorpus = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+  staticChecks: boolean,
+): Promise<CorpusSelection> => {
+  if (!staticChecks) {
+    const corpus = await prepareCorpus(execution);
+    await evaluationIdentity(execution, corpus, mode);
+    return corpus;
+  }
+  const corpus = await inspectCorpus(execution);
+  const campaign = execution.loaded.campaign;
+  for (const profileId of campaign.profiles) profileOrThrow(execution.config, profileId);
+  if (corpus.kind === "agent") await validateAgentCaseProfileDrivers(execution, corpus);
+  for (const validation of campaign.subject.validate)
+    assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+  return corpus;
+};

package/src/orchestrator/experiment.ts

@@ -48,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -182,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -192,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({