@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.32

package/src/orchestrator/comparison-identity.ts

@@ -31,10 +31,12 @@ export const runComparisonIdentity = (metadata: RunMetadata): string =>
   valueHash({
     source: sourceIdentity(metadata),
     runtime: metadata.provenance?.runtime,
+    environment: metadata.provenance?.environment,
     caseHash: metadata.provenance?.caseHash,
     assertionsHash: metadata.provenance?.assertionsHash,
     fixturesHash: metadata.provenance?.fixturesHash,
     profileId: metadata.provenance?.profileId,
+    permutationId: metadata.provenance?.permutationId,
     driverConfigHash: metadata.provenance?.driverConfigHash,
     answererConfigHash: metadata.provenance?.answererConfigHash,
     evaluatorConfigHash: metadata.provenance?.evaluatorConfigHash,

package/src/orchestrator/config.ts

@@ -4,11 +4,11 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
 import { createRubricAssertion } from "../assertions/rubric/index.js";
+import { EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type AnswererPlugin,
   AnswererRefSchema,
@@ -18,13 +18,18 @@ import {
   type DriverPlugin,
   DriverRefSchema,
   formatZodIssues,
+  HarnessMatrixSchema,
   HarnessModelConfigSchema,
   type LifecycleHooksPlugin,
   lifecycleHooksPluginIssue,
   Registry,
 } from "../core/index.js";
+import { antigravityDriver } from "../drivers/antigravity/index.js";
 import { claudeCodeDriver } from "../drivers/claude-code/index.js";
 import { codexDriver } from "../drivers/codex/index.js";
+import { copilotDriver } from "../drivers/copilot/index.js";
+import { cursorDriver } from "../drivers/cursor/index.js";
+import { grokBuildDriver } from "../drivers/grok-build/index.js";
 import { mcpProbeDriver } from "../drivers/mcp/index.js";
 import { subprocessDriver } from "../drivers/subprocess/index.js";
 
@@ -35,6 +40,7 @@ const pluginLike = z.looseObject({
 
 export const ExperimentProfileSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     driver: DriverRefSchema.optional(),
     answerer: AnswererRefSchema.optional(),
     harness: HarnessModelConfigSchema.optional(),
@@ -68,8 +74,9 @@ const LifecycleHooksPluginSchema = z.custom<LifecycleHooksPlugin>().superRefine(
   if (issue) ctx.addIssue({ code: "custom", message: issue });
 });
 
-const LuxConfigSchema = z
+export const LuxConfigSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     casesRoot: z.string().optional(),
     fixturesRoot: z.string().optional(),
     runsRoot: z.string().default(".lux-runs"),
@@ -77,6 +84,8 @@ const LuxConfigSchema = z
     subjectRoot: z.string().optional(),
     defaultAnswerer: AnswererRefSchema.default("persona"),
     defaultDriver: DriverRefSchema.optional(),
+    /** Driver, model, and effort permutations every case runs through by default. */
+    matrix: HarnessMatrixSchema.optional(),
     /** Shared provider/model defaults for persona answering and rubric judging. */
     harness: HarnessModelConfigSchema.optional(),
     concurrency: z.number().int().min(1).default(1),
@@ -163,8 +172,10 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
     ...(validated.fixturesRoot ? { fixturesRoot: resolve(root, validated.fixturesRoot) } : {}),
     defaultAnswerer: validated.defaultAnswerer,
     ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
+    ...(validated.matrix ? { matrix: validated.matrix } : {}),
     ...(validated.harness ? { harness: validated.harness } : {}),
     concurrency: validated.concurrency,
+    ...(validated.environment ? { environment: validated.environment } : {}),
     ...(validated.profiles ? { profiles: validated.profiles } : {}),
     ...(validated.drivers ? { drivers: validated.drivers } : {}),
     ...(validated.answerers ? { answerers: validated.answerers } : {}),
@@ -176,13 +187,19 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
 const builtinDrivers: DriverPlugin<never>[] = [
   claudeCodeDriver,
   codexDriver,
+  cursorDriver,
+  copilotDriver,
+  grokBuildDriver,
+  antigravityDriver,
   subprocessDriver,
   mcpProbeDriver,
 ];
+export const isBuiltinDriver = (plugin: DriverPlugin): boolean =>
+  builtinDrivers.some((builtin) => builtin === plugin);
+
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {

package/src/orchestrator/discover.ts

@@ -13,6 +13,7 @@ export type DiscoveredCase = {
   relPath: string;
   /** `meta.tags` normalized to strings; empty when unset. */
   tags: string[];
+  cluster?: string;
 };
 
 /**
@@ -96,7 +97,13 @@ export const discoverCases = async (root: string): Promise<DiscoveryResult> => {
       problems.push({ path, reason: formatZodIssues(parsed.error) });
       continue;
     }
-    cases.push({ path, id: parsed.data.id ?? relPath, relPath, tags: tagsOf(parsed.data.meta) });
+    cases.push({
+      path,
+      id: parsed.data.id ?? relPath,
+      relPath,
+      tags: tagsOf(parsed.data.meta),
+      ...(parsed.data.meta?.cluster ? { cluster: parsed.data.meta.cluster } : {}),
+    });
   }
   return { cases, problems };
 };

package/src/orchestrator/doctor.ts

@@ -4,17 +4,34 @@ import { resolve } from "node:path";
 import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
-import type { EvalCase, Registry } from "../core/index.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
+import {
+  casePermutations,
+  type EvalCase,
+  expandHarnessMatrix,
+  type HarnessPermutation,
+  permutationDriverRef,
+  permutationId,
+  type Registry,
+} from "../core/index.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
-import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
+import { validateDriverCapabilities } from "./driver-capabilities.js";
+import { ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
 import {
   experimentCampaignMode,
   loadExperimentCampaign,
   resolveExperimentCampaignPath,
 } from "./experiment-loader.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
 import { assertionConfigForSchema } from "./grading.js";
+import {
+  describeHarnessCatalogDrift,
+  harnessCatalogDrift,
+  harnessCatalogProbes,
+  hasHarnessCatalogDrift,
+} from "./harness-catalog-drift.js";
 import { loadEvalCase } from "./loader.js";
 import { validatePluginConfig, validatePluginConfigAsync } from "./validate-plugin-config.js";
 
@@ -60,6 +77,7 @@ export const DoctorOptionsSchema = z
   .object({
     cwd: z.string().min(1),
     campaigns: z.array(z.string()).optional(),
+    static: z.boolean().optional(),
   })
   .strict();
 export type DoctorOptions = z.input<typeof DoctorOptionsSchema>;
@@ -126,10 +144,32 @@ const effectiveDriverReference = (local: unknown, fallback: unknown): unknown =>
   return {
     name: refName(local),
     config: mergeRecords(rawRefConfig(fallback), rawRefConfig(local)),
+    execution: driverExecutionOf(local) ?? driverExecutionOf(fallback),
   };
 };
 
-const caseDriverReferences = (local: unknown, config: ResolvedLuxConfig): unknown[] => {
+const boundPermutation = (
+  permutation: HarnessPermutation,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): unknown => {
+  const plugin = registry.getDriver(permutation.driver);
+  if (!plugin) throw new Error(`unknown driver '${permutation.driver}'`);
+  return permutationDriverRef(permutation, config.defaultDriver, plugin);
+};
+
+const caseDriverReferences = (
+  authored: Pick<EvalCase, "driver" | "matrix">,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): unknown[] => {
+  const local = authored.driver;
+  const permutations = casePermutations(authored, config.matrix);
+  if (permutations.length > 0) {
+    return permutations.map((permutation) =>
+      effectiveDriverReference(local, boundPermutation(permutation, config, registry)),
+    );
+  }
   const fallbacks = [
     config.defaultDriver,
     ...Object.values(config.profiles ?? {}).map((profile) => profile.driver),
@@ -142,17 +182,20 @@ const caseDriverReferences = (local: unknown, config: ResolvedLuxConfig): unknow
   ];
 };
 
-const driverCommand = (reference: unknown): string | undefined => {
+const driverCommands = (reference: unknown, registry: Registry): string[] => {
   const name = refName(reference);
-  const config = refConfig(reference);
-  if (name === "claude-code") {
-    return typeof config.command === "string" ? config.command : "claude";
-  }
-  if (name === "codex") return typeof config.command === "string" ? config.command : "codex";
-  if (name === "subprocess") {
-    return typeof config.command === "string" ? config.command : undefined;
+  const plugin = name ? registry.getDriver(name) : undefined;
+  if (!plugin?.runtime) return [];
+  try {
+    const config = validatePluginConfig(
+      plugin.configSchema,
+      rawRefConfig(reference),
+      `driver '${name}'`,
+    );
+    return plugin.runtime(config).commands;
+  } catch {
+    return [];
   }
-  return undefined;
 };
 
 const answererCommand = (reference: unknown): string | undefined => {
@@ -179,12 +222,24 @@ const rubricCommands = (assertion: unknown, config: ResolvedLuxConfig): string[]
   return [...new Set(commands)];
 };
 
-const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
+const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): CliTarget[] => {
   const targets: CliTarget[] = [];
   const add = (label: string, command: string | undefined): void => {
     if (command) targets.push({ label, command });
   };
-  add("default driver", driverCommand(config.defaultDriver));
+  for (const command of driverCommands(config.defaultDriver, registry))
+    add("default driver", command);
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    try {
+      for (const command of driverCommands(
+        boundPermutation(permutation, config, registry),
+        registry,
+      ))
+        add(`matrix '${permutationId(permutation)}' driver`, command);
+    } catch {
+      /* Reported by validateConfiguredMatrix. */
+    }
+  }
   add("default answerer", answererCommand(config.defaultAnswerer));
   if (config.harness) {
     add(
@@ -195,7 +250,9 @@ const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
   for (const [id, profile] of Object.entries(config.profiles ?? {}).sort(([a], [b]) =>
     a.localeCompare(b),
   )) {
-    add(`profile '${id}' driver`, driverCommand(profile.driver ?? config.defaultDriver));
+    for (const command of driverCommands(profile.driver ?? config.defaultDriver, registry)) {
+      add(`profile '${id}' driver`, command);
+    }
     add(`profile '${id}' answerer`, answererCommand(profile.answerer ?? config.defaultAnswerer));
     const harness = profile.harness ?? config.harness;
     if (harness) {
@@ -348,6 +405,32 @@ const validateConfiguredDrivers = (
   }
 };
 
+const validateConfiguredMatrix = (
+  config: ResolvedLuxConfig,
+  registry: Registry,
+  recorder: DoctorRecorder,
+): void => {
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    const id = permutationId(permutation);
+    try {
+      const reference = boundPermutation(permutation, config, registry);
+      const plugin = registry.getDriver(permutation.driver);
+      validatePluginConfig(
+        plugin?.configSchema,
+        rawRefConfig(reference),
+        `matrix '${id}' driver '${permutation.driver}'`,
+      );
+    } catch (error) {
+      recorder.add(
+        `matrix:${id}`,
+        "fail",
+        `matrix permutation '${id}' is not runnable`,
+        firstLine(error),
+      );
+    }
+  }
+};
+
 const validateConfiguredAnswerers = (
   config: ResolvedLuxConfig,
   registry: Registry,
@@ -418,17 +501,19 @@ const caseDriverFailure = (
   caseId: string,
   reference: unknown,
   registry: Registry,
+  required: EvalCase["requires"],
 ): string | null => {
   const name = refName(reference);
   if (!name) return "driver reference has no name";
   const plugin = registry.getDriver(name);
   if (!plugin) return `unknown driver '${name}'`;
   try {
-    validatePluginConfig(
+    const config = validatePluginConfig(
       plugin.configSchema,
       rawRefConfig(reference),
       `case '${caseId}' driver '${name}'`,
     );
+    validateDriverCapabilities(plugin, config, required, driverExecutionOf(reference));
     return null;
   } catch (error) {
     return firstLine(error);
@@ -440,6 +525,7 @@ const validateCaseDrivers = (
   drivers: readonly unknown[],
   registry: Registry,
   recorder: DoctorRecorder,
+  required: EvalCase["requires"],
 ): void => {
   if (drivers.length === 0) {
     recorder.add(
@@ -450,7 +536,7 @@ const validateCaseDrivers = (
     return;
   }
   const failures = drivers
-    .map((driver) => caseDriverFailure(caseId, driver, registry))
+    .map((driver) => caseDriverFailure(caseId, driver, registry, required))
     .filter((failure): failure is string => failure !== null);
   if (failures.length === drivers.length) {
     recorder.add(
@@ -542,13 +628,16 @@ const executionTargetsForCase = (
   caseId: string,
   drivers: readonly unknown[],
   answerer: unknown,
+  registry: Registry,
 ): CliTarget[] => {
   const targets = drivers.flatMap((driver): CliTarget[] => {
-    const command = driverCommand(driver);
-    return command ? [{ label: `case '${caseId}' driver`, command }] : [];
+    return driverCommands(driver, registry).map((command) => ({
+      label: `case '${caseId}' driver`,
+      command,
+    }));
   });
-  const command = answererCommand(answerer);
-  if (command) targets.push({ label: `case '${caseId}' answerer`, command });
+  const answererCli = answererCommand(answerer);
+  if (answererCli) targets.push({ label: `case '${caseId}' answerer`, command: answererCli });
   return targets;
 };
 
@@ -561,13 +650,13 @@ const inspectCase = async (
   try {
     const authored = await loadEvalCase(discovered.path);
     const caseRegistry = registryForEvalCase(registry, authored);
-    const drivers = caseDriverReferences(authored.driver, config);
+    const drivers = caseDriverReferences(authored, config, caseRegistry);
     const answerer = authored.answerer ?? config.defaultAnswerer;
-    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
+    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder, authored.requires);
     validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [
       ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
-      ...executionTargetsForCase(discovered.id, drivers, answerer),
+      ...executionTargetsForCase(discovered.id, drivers, answerer, caseRegistry),
     ];
   } catch (error) {
     recorder.add(
@@ -644,6 +733,54 @@ const inspectCliTargets = async (
   }
 };
 
+const catalogDriverReferences = (config: ResolvedLuxConfig): unknown[] => [
+  ...configuredDriverReferences(config).map(([, reference]) => reference),
+  ...(config.matrix ?? []).map((entry) => ({ name: entry.driver, config: entry.config })),
+];
+
+const configuredDriverCommand = (config: ResolvedLuxConfig, driver: string): string | undefined => {
+  for (const reference of catalogDriverReferences(config)) {
+    if (refName(reference) !== driver) continue;
+    const command = (rawRefConfig(reference) as { command?: unknown } | undefined)?.command;
+    if (typeof command === "string") return command;
+  }
+  return undefined;
+};
+
+const inspectHarnessCatalogs = async (
+  config: ResolvedLuxConfig,
+  recorder: DoctorRecorder,
+): Promise<void> => {
+  const configured = new Set(
+    catalogDriverReferences(config)
+      .map(refName)
+      .filter((name): name is string => name !== undefined),
+  );
+  for (const probe of harnessCatalogProbes) {
+    const driver = probe.catalog.driver;
+    if (!configured.has(driver)) continue;
+    const live = await probe.read(configuredDriverCommand(config, driver));
+    if (!live) {
+      recorder.add(
+        `catalog:${driver}`,
+        "warn",
+        `${driver} harness catalog (verified ${probe.catalog.verified}) could not be compared with the installed CLI`,
+        `${probe.source} was unavailable`,
+      );
+      continue;
+    }
+    const drift = harnessCatalogDrift(probe.catalog, live);
+    recorder.add(
+      `catalog:${driver}`,
+      hasHarnessCatalogDrift(drift) ? "warn" : "pass",
+      hasHarnessCatalogDrift(drift)
+        ? `${driver} harness catalog (verified ${probe.catalog.verified}) differs from the installed CLI`
+        : `${driver} harness catalog matches the installed CLI`,
+      describeHarnessCatalogDrift(drift) || undefined,
+    );
+  }
+};
+
 const isBlockingCampaignWarning = (warning: string): boolean =>
   warning.includes("below confidence minSamples=") || warning.includes("cannot attain confidence");
 
@@ -652,6 +789,7 @@ const inspectCampaign = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   try {
     const loaded = await loadExperimentCampaign(path);
@@ -665,7 +803,9 @@ const inspectCampaign = async (
         placeholders.join(", "),
       );
     }
-    const plan = await planExperimentCampaign({ config, registry, loaded }, mode);
+    const plan = await planExperimentCampaign({ config, registry, loaded }, mode, {
+      static: staticChecks,
+    });
     recorder.campaigns.push({ path, campaignId: loaded.campaign.id, mode, plan });
     recorder.add(
       `campaign:${loaded.campaign.id}`,
@@ -697,6 +837,7 @@ const inspectCampaigns = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   let paths: string[];
   try {
@@ -719,7 +860,7 @@ const inspectCampaigns = async (
     );
   }
   for (const path of paths) {
-    await inspectCampaign(path, config, registry, recorder);
+    await inspectCampaign(path, config, registry, recorder, staticChecks);
   }
 };
 
@@ -733,10 +874,30 @@ export const doctorProject = async (options: DoctorOptions): Promise<DoctorRepor
   if (!registry) return recorder.finish(config.rootDir);
 
   validateConfiguredDrivers(config, registry, recorder);
+  validateConfiguredMatrix(config, registry, recorder);
   validateConfiguredAnswerers(config, registry, recorder);
   const caseCliTargets = await inspectCases(config, registry, recorder);
-  await inspectCliTargets([...configuredCliTargets(config), ...caseCliTargets], config, recorder);
-  await inspectCampaigns(parsedOptions.campaigns ?? [], config, registry, recorder);
+  if (parsedOptions.static) {
+    recorder.add(
+      "static-checks",
+      "warn",
+      "static checks only; installed executables and campaign execution identity were not checked",
+    );
+  } else {
+    await inspectCliTargets(
+      [...configuredCliTargets(config, registry), ...caseCliTargets],
+      config,
+      recorder,
+    );
+    await inspectHarnessCatalogs(config, recorder);
+  }
+  await inspectCampaigns(
+    parsedOptions.campaigns ?? [],
+    config,
+    registry,
+    recorder,
+    parsedOptions.static ?? false,
+  );
   return recorder.finish(config.rootDir);
 };
 

package/src/orchestrator/driver-capabilities.ts

@@ -0,0 +1,21 @@
+import type { DriverCapability, DriverPlugin } from "../core/driver.js";
+import type { DriverExecution } from "../core/driver-execution.js";
+
+export const validateDriverCapabilities = (
+  plugin: DriverPlugin,
+  config: unknown,
+  required: readonly DriverCapability[] = [],
+  execution?: DriverExecution,
+): void => {
+  if (required.length === 0) return;
+  const capabilities = {
+    ...plugin.runtime?.(config).capabilities,
+    ...(execution?.kind === "container" ? { isolation: true } : {}),
+  };
+  const missing = required.filter((capability) => capabilities?.[capability] !== true);
+  if (missing.length > 0) {
+    throw new Error(
+      `driver '${plugin.name}' does not declare required capabilities: ${missing.join(", ")}`,
+    );
+  }
+};

package/src/orchestrator/driver-identity.ts

@@ -0,0 +1,69 @@
+import type { DriverPlugin, DriverRuntime } from "../core/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
+import type { Registry } from "../core/registry.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+
+const resolvedDriver = (
+  reference: unknown,
+  registry: Registry,
+): { plugin: DriverPlugin; config: unknown } | undefined => {
+  const name =
+    typeof reference === "string" ? reference : (reference as { name?: unknown } | null)?.name;
+  if (typeof name !== "string") return undefined;
+  const plugin = registry.getDriver(name);
+  if (!plugin) throw new Error(`unknown driver '${name}'`);
+  const raw =
+    typeof reference === "object" && reference !== null
+      ? (reference as { config?: unknown }).config
+      : undefined;
+  return { plugin, config: validatePluginConfig(plugin.configSchema, raw, `driver '${name}'`) };
+};
+
+export const referencedDriverRuntime = (reference: unknown, registry: Registry): DriverRuntime => {
+  const resolved = resolvedDriver(reference, registry);
+  const runtime = resolved?.plugin.runtime?.(resolved.config) ?? { commands: [], capabilities: {} };
+  const execution = driverExecutionOf(reference);
+  return execution?.kind === "container"
+    ? {
+        commands: [execution.command ?? "docker"],
+        capabilities: { ...runtime.capabilities, isolation: true },
+      }
+    : runtime;
+};
+
+export const experimentDriverIdentity = async (
+  reference: unknown,
+  registry: Registry,
+  rootDir: string,
+): Promise<unknown> => {
+  if (driverExecutionOf(reference)?.kind === "container")
+    throw new Error("container execution does not yet support controlled experiments");
+  const resolved = resolvedDriver(reference, registry);
+  if (!resolved) return null;
+  const { plugin, config } = resolved;
+  const experiment = plugin.experiment;
+  if (!experiment)
+    throw new Error(
+      `driver '${plugin.name}' supports ordinary eval runs but does not declare controlled experiment support`,
+    );
+  if (!experiment.version.trim())
+    throw new Error(`driver '${plugin.name}' experiment version must not be empty`);
+  const prepared = validatePluginConfig(
+    plugin.configSchema,
+    experiment.prepare(config),
+    `driver '${plugin.name}' experiment configuration`,
+  );
+  const runtime = plugin.runtime?.(prepared) ?? { commands: [], capabilities: {} };
+  const behavior = await experiment.identity(config, { rootDir });
+  return {
+    driver: plugin.name,
+    version: experiment.version,
+    config: prepared,
+    capabilities: runtime.capabilities,
+    executables: await Promise.all(
+      runtime.commands.map((command) => cliIdentity(command, rootDir)),
+    ),
+    behavior,
+  };
+};

package/src/orchestrator/experiment-context.ts

@@ -3,17 +3,18 @@ import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
 
-export type EvaluationCorpus =
+export type CorpusSelection =
   | {
       kind: "agent";
       cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
-      hash: string;
     }
   | {
       kind: "grader";
       hash: string;
     };
 
+export type EvaluationCorpus = CorpusSelection & { hash: string };
+
 export type CampaignExecutionOptions = {
   config: ResolvedLuxConfig;
   registry: Registry;