@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/orchestrator.ts

@@ -1,10 +1,15 @@
-import { basename, extname, isAbsolute, join, resolve } from "node:path";
+import { basename, extname, join, resolve } from "node:path";
+import { ContainerDriver } from "../container/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
+import { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type Answerer,
   type AnswererModelCallAccounting,
   type AnswererRef,
   bindSubjectRoot,
   type DriverRef,
+  driverConfigOf,
+  driverNameOf,
   type EvalCase,
   type GradingAssertionEntry,
   type GradingReport,
@@ -18,12 +23,20 @@ import {
   type SuiteLifecycleHookContext,
   valueHash,
 } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
+import {
+  mergeAnswererReference,
+  mergeDriverReference,
+  withDriverDefaults,
+} from "../core/reference-defaults.js";
 import {
   fixturesForEvalCase,
   mergeLifecycleFixtures,
   registryForEvalCase,
   valuesForFixtures,
 } from "./case-extensions.js";
+import { isBuiltinDriver } from "./config.js";
+import { validateDriverCapabilities } from "./driver-capabilities.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
   assertionEvaluatorIdentities,
@@ -102,11 +115,13 @@ const recordLifecycleCleanupFailure = async (result: RunResult, error: unknown):
 };
 
 export type OrchestratorOptions = {
+  environment?: EnvironmentDeclaration | undefined;
   registry: Registry;
   /** Answerer used when a case names none. Defaults to "persona". */
   defaultAnswerer?: AnswererRef;
   /** Driver used when a case names none. */
   defaultDriver?: DriverRef;
+  driverDefaults?: Record<string, unknown> | undefined;
   /** Shared provider/model defaults for built-in harness components. */
   harness?: HarnessModelConfig;
   /** Versioned project resource fixtures. */
@@ -114,6 +129,8 @@ export type OrchestratorOptions = {
 };
 
 export type RunRequest = {
+  /** Effective parent suite worker limit; direct runs default to one. */
+  suiteConcurrency?: number | undefined;
   /** Path to a TS case file. Mutually exclusive with `case`. */
   casePath?: string;
   /** Pre-loaded EvalCase. Mutually exclusive with `casePath`. */
@@ -137,6 +154,12 @@ export type RunRequest = {
   subjectRoot?: string | undefined;
   subjectHash?: string | undefined;
   profileId?: string | undefined;
+  /**
+   * Driver bound by a harness permutation. It replaces the project default;
+   * a case pinned to another driver is rejected rather than silently rerouted.
+   */
+  driver?: DriverRef | undefined;
+  permutationId?: string | undefined;
   experimentId?: string | undefined;
   variantId?: string | undefined;
   candidateId?: string | undefined;
@@ -195,7 +218,13 @@ export type SuiteObserver = RunObserver & {
  * `caseId` is the canonical id runs are filed under when the case declares
  * none — display decorations (a `--loop` suffix) must not fork run history.
  */
-export type SuiteItem = { path: string; id: string; caseId?: string | undefined };
+export type SuiteItem = {
+  path: string;
+  id: string;
+  caseId?: string | undefined;
+  driver?: DriverRef | undefined;
+  permutationId?: string | undefined;
+};
 
 const resolveCaseId = (casePath: string, evalCase: EvalCase, fallback?: string): string => {
   if (evalCase.id) return evalCase.id;
@@ -206,40 +235,24 @@ const resolveCaseId = (casePath: string, evalCase: EvalCase, fallback?: string):
   return ext ? base.slice(0, -ext.length) : base;
 };
 
-const driverNameOf = (ref: DriverRef): string => (typeof ref === "string" ? ref : ref.name);
-
-const isRecord = (value: unknown): value is Record<string, unknown> =>
-  Boolean(value) && typeof value === "object" && !Array.isArray(value);
-
-/** Merge plugin config recursively so nested Codex `config`/`env` overrides stay additive. */
-const mergePluginConfig = (base: unknown, local: unknown): unknown => {
-  if (!isRecord(base) || !isRecord(local)) return local ?? base;
-  const merged: Record<string, unknown> = { ...base };
-  for (const [key, value] of Object.entries(local)) {
-    merged[key] = key in merged ? mergePluginConfig(merged[key], value) : value;
-  }
-  return merged;
-};
-
 export const resolveDriverReference = (
   evalCase: EvalCase,
   fallback: DriverRef | undefined,
+  permutation?: DriverRef | undefined,
+  defaults?: Record<string, unknown> | undefined,
 ): DriverRef => {
   const local = evalCase.driver;
-  if (!local) {
-    if (!fallback) throw new Error("case names no driver and no defaultDriver is configured");
-    return fallback;
+  if (permutation && local && driverNameOf(local) !== driverNameOf(permutation)) {
+    throw new Error(
+      `case is pinned to driver '${driverNameOf(local)}' but the run selected '${driverNameOf(permutation)}'`,
+    );
   }
-  if (!fallback || driverNameOf(local) !== driverNameOf(fallback)) return local;
-
-  // Naming the same driver opts into project defaults. A case object only
-  // needs to carry the fields it wants to override (for example a timeout).
-  const baseConfig = typeof fallback === "object" ? fallback.config : undefined;
-  const localConfig = typeof local === "object" ? local.config : undefined;
-  return {
-    name: driverNameOf(local),
-    config: mergePluginConfig(baseConfig, localConfig),
-  };
+  const reference = withDriverDefaults(
+    mergeDriverReference(permutation ?? fallback, local),
+    defaults,
+  );
+  if (!reference) throw new Error("case names no driver and no defaultDriver is configured");
+  return reference;
 };
 
 export const resolveAnswererReference = (
@@ -248,7 +261,18 @@ export const resolveAnswererReference = (
   harness: HarnessModelConfig | undefined,
   override?: string,
 ): AnswererRef => {
-  const ref = override ?? evalCase.answerer ?? fallback;
+  const fallbackName = typeof fallback === "string" ? fallback : fallback.name;
+  const inherited =
+    harness && fallbackName === "persona"
+      ? {
+          name: "persona",
+          config: mergeHarnessModelConfig(
+            harness,
+            typeof fallback === "object" ? fallback.config : undefined,
+          ),
+        }
+      : fallback;
+  const ref = mergeAnswererReference(inherited, override ?? evalCase.answerer) as AnswererRef;
   const name = typeof ref === "string" ? ref : ref.name;
   if (!harness || name !== "persona") return ref;
   return {
@@ -259,197 +283,25 @@ export const resolveAnswererReference = (
 
 const answererNameOf = (ref: AnswererRef): string => (typeof ref === "string" ? ref : ref.name);
 
-const driverConfigOf = (ref: DriverRef): unknown =>
-  typeof ref === "object" ? ref.config : undefined;
-
 const answererConfigOf = (ref: AnswererRef): unknown =>
   typeof ref === "object" ? ref.config : undefined;
 
-const EXPERIMENT_BEHAVIOR_SOURCE_ARGS = new Set([
-  "--continue",
-  "-c",
-  "--fork-session",
-  "--resume",
-  "-r",
-  "--add-dir",
-  "--append-system-prompt-file",
-  "--mcp-config",
-  "--plugin-dir",
-  "--settings",
-  "--setting-sources",
-  "--system-prompt-file",
-]);
-
-const CODEX_EXPERIMENT_SOURCE_ARGS = new Set([
-  "--add-dir",
-  "--cd",
-  "-C",
-  "--config",
-  "-c",
-  "--profile",
-  "-p",
-]);
-
-const bindSubprocessExperimentConfig = (config: Record<string, unknown>): unknown => {
-  if (typeof config.command !== "string" || !isAbsolute(config.command)) {
-    throw new Error(
-      "experiment subprocess commands must be absolute direct wrappers so Lux can fingerprint the invoked behavior",
-    );
-  }
-  if (typeof config.cwd === "string") {
-    throw new Error(
-      "experiment runs do not allow an explicit subprocess cwd; use the candidate subject and staged fixtures",
-    );
-  }
-  if (Array.isArray(config.args) && config.args.length > 0) {
-    throw new Error(
-      "experiment subprocesses do not allow args; put stable behavior in the fingerprinted direct wrapper",
-    );
-  }
-  if (isRecord(config.env) && Object.keys(config.env).length > 0) {
-    throw new Error(
-      "experiment subprocesses do not allow custom env; put stable behavior in the fingerprinted direct wrapper",
-    );
-  }
-  return { ...config, inheritEnv: false };
-};
-
-type AgentDriverName = "claude-code" | "codex";
-
-const validateAgentExperimentConfig = (
-  driverName: AgentDriverName,
-  config: Record<string, unknown>,
-): { driverEnv: Record<string, unknown>; extraArgs: string[] } => {
-  const driverEnv = isRecord(config.env) ? config.env : {};
-  const commandResolutionOverride = Object.keys(driverEnv).find((name) =>
-    ["PATH", "PATHEXT"].includes(name.toUpperCase()),
-  );
-  if (commandResolutionOverride) {
-    throw new Error(
-      `experiment runs do not allow driver env.${commandResolutionOverride}; command resolution must match the fingerprinted host executable`,
-    );
-  }
-  if (typeof config.cwd === "string") {
-    throw new Error(
-      `experiment runs do not allow an explicit ${driverName} cwd; use the candidate subject and staged fixtures`,
-    );
-  }
-
-  const extraArgs = Array.isArray(config.extraArgs)
-    ? config.extraArgs.filter((value): value is string => typeof value === "string")
-    : [];
-  const disallowedArgs =
-    driverName === "claude-code" ? EXPERIMENT_BEHAVIOR_SOURCE_ARGS : CODEX_EXPERIMENT_SOURCE_ARGS;
-  const behaviorSourceArg = extraArgs.find((arg) =>
-    [...disallowedArgs].some((name) => arg === name || arg.startsWith(`${name}=`)),
-  );
-  if (behaviorSourceArg) {
-    throw new Error(
-      `experiment runs do not allow behavior-bearing extraArgs '${behaviorSourceArg}'; use structured driver fields so Lux can fingerprint the source`,
-    );
-  }
-
-  const isolation = isRecord(config.isolation) ? config.isolation : {};
-  const structuredPaths = [
-    ...(driverName === "claude-code" && Array.isArray(config.pluginDirs) ? config.pluginDirs : []),
-    ...(Array.isArray(config.addDirs) ? config.addDirs : []),
-    ...(driverName === "claude-code" && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead
-      : []),
-  ].filter((value): value is string => typeof value === "string");
-  const relativeSource = structuredPaths.find((path) => !isAbsolute(path));
-  if (relativeSource) {
-    throw new Error(
-      `experiment driver source paths must be absolute after subjectPath binding: ${relativeSource}`,
-    );
-  }
-  return { driverEnv, extraArgs };
-};
-
-const bindCodexExperimentConfig = (
-  config: Record<string, unknown>,
-  extraArgs: readonly string[],
-): unknown => {
-  if (typeof config.profile === "string") {
-    throw new Error(
-      "experiment runs do not allow a Codex profile; use inline driver config so Lux can fingerprint it",
-    );
-  }
-  if (config.ephemeral === false) {
-    throw new Error("experiment runs require ephemeral Codex sessions");
-  }
-  const inlineConfig = isRecord(config.config) ? config.config : {};
-  const externalConfigKey = ((): string | undefined => {
-    const visit = (value: Record<string, unknown>, parent = ""): string | undefined => {
-      for (const [key, nested] of Object.entries(value)) {
-        const path = parent ? `${parent}.${key}` : key;
-        if (/(?:mcp_servers|instructions_file|notify|project_doc)/i.test(key)) return path;
-        if (isRecord(nested)) {
-          const found = visit(nested, path);
-          if (found) return found;
-        }
-      }
-      return undefined;
-    };
-    return visit(inlineConfig);
-  })();
-  if (externalConfigKey) {
-    throw new Error(
-      `experiment runs do not allow path-bearing Codex config '${externalConfigKey}'`,
-    );
-  }
-  const appServer = config.interactionMode !== "exec";
-  return {
-    ...config,
-    ephemeral: true,
-    ignoreUserConfig: true,
-    ignoreRules: true,
-    ...(appServer && !config.isolation ? { isolation: { allowRead: [] } } : {}),
-    extraArgs,
-  };
-};
-
-const bindClaudeExperimentConfig = (
-  config: Record<string, unknown>,
-  extraArgs: readonly string[],
-  driverEnv: Record<string, unknown>,
-): unknown => {
-  if (config.isolation) return config;
-  return {
-    ...config,
-    extraArgs: [
-      ...extraArgs,
-      "--setting-sources",
-      "user",
-      "--strict-mcp-config",
-      "--settings",
-      JSON.stringify({ disableAllHooks: true }),
-    ],
-    env: {
-      ...driverEnv,
-      CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",
-      CLAUDE_CODE_DISABLE_CLAUDE_MDS: "1",
-      CLAUDE_CODE_SKIP_PROMPT_HISTORY: "1",
-      CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1",
-      CLAUDE_CODE_AUTO_CONNECT_IDE: "false",
-      CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL: "1",
-      DISABLE_AUTOUPDATER: "1",
-    },
-  };
-};
-
 export const boundExperimentDriverConfig = (
-  driverName: string,
+  plugin: import("../core/driver.js").DriverPlugin,
   config: unknown,
   experimentId: string | undefined,
 ): unknown => {
-  if (!experimentId || !isRecord(config)) return config;
-  if (driverName === "subprocess") return bindSubprocessExperimentConfig(config);
-  if (driverName !== "claude-code" && driverName !== "codex") return config;
-  const { driverEnv, extraArgs } = validateAgentExperimentConfig(driverName, config);
-  return driverName === "codex"
-    ? bindCodexExperimentConfig(config, extraArgs)
-    : bindClaudeExperimentConfig(config, extraArgs, driverEnv);
+  if (!experimentId) return config;
+  if (!plugin.experiment) {
+    throw new Error(
+      `driver '${plugin.name}' supports ordinary eval runs but does not declare controlled experiment support`,
+    );
+  }
+  return validatePluginConfig(
+    plugin.configSchema,
+    plugin.experiment.prepare(config),
+    `driver '${plugin.name}' experiment configuration`,
+  );
 };
 
 const closeQuietly = async (close: () => Promise<void>, label: string): Promise<void> => {
@@ -463,17 +315,23 @@ const closeQuietly = async (close: () => Promise<void>, label: string): Promise<
 };
 
 export class Orchestrator {
+  private readonly environment: EnvironmentDeclaration | undefined;
   private evaluatorHash: Promise<string> | undefined;
   readonly registry: Registry;
   private readonly defaultAnswerer: AnswererRef;
   private readonly defaultDriver: DriverRef | undefined;
+  private readonly driverDefaults: Record<string, unknown> | undefined;
   private readonly harness: HarnessModelConfig | undefined;
   private readonly lifecycleFixtures: readonly LifecycleHooksPlugin[];
 
   constructor(options: OrchestratorOptions) {
+    this.environment = options.environment
+      ? EnvironmentDeclarationSchema.parse(options.environment)
+      : undefined;
     this.registry = options.registry;
     this.defaultAnswerer = options.defaultAnswerer ?? "persona";
     this.defaultDriver = options.defaultDriver;
+    this.driverDefaults = options.driverDefaults;
     this.harness = options.harness;
     this.lifecycleFixtures = [...(options.lifecycleFixtures ?? [])];
   }
@@ -512,13 +370,15 @@ export class Orchestrator {
       registry,
     );
 
-    const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
+    const driverRef = bindSubjectRoot(
+      resolveDriverReference(evalCase, this.defaultDriver, request.driver, this.driverDefaults),
+      request.subjectRoot,
+    );
     const driverName = driverNameOf(driverRef);
-    const answererRef = resolveAnswererReference(
-      evalCase,
-      this.defaultAnswerer,
-      this.harness,
-      request.answererOverride,
+    const harness = resolveHarnessForDriver(this.harness, driverRef);
+    const answererRef = bindSubjectRoot(
+      resolveAnswererReference(evalCase, this.defaultAnswerer, harness, request.answererOverride),
+      request.subjectRoot,
     );
     const answererName = answererNameOf(answererRef);
 
@@ -534,7 +394,7 @@ export class Orchestrator {
     // Validate both configs before instantiating either plugin so a
     // schema failure can't leak a half-initialized driver/answerer.
     const driverCfg = boundExperimentDriverConfig(
-      driverName,
+      driverPlugin,
       validatePluginConfig(
         driverPlugin.configSchema,
         driverConfigOf(driverRef),
@@ -548,7 +408,20 @@ export class Orchestrator {
       `answerer '${answererName}'`,
     );
 
-    const driver = await driverPlugin.create(driverCfg);
+    const execution = driverExecutionOf(driverRef);
+    if (execution?.kind === "container" && request.experimentId)
+      throw new Error("container execution does not yet support controlled experiments");
+    if (
+      execution?.kind === "container" &&
+      !execution.driverModule &&
+      !isBuiltinDriver(driverPlugin)
+    )
+      throw new Error("custom container drivers require execution.driverModule in the image");
+    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires, execution);
+    const driver =
+      execution?.kind === "container"
+        ? new ContainerDriver(driverName, driverCfg, execution)
+        : await driverPlugin.create(driverCfg);
     let answerer: Answerer;
     try {
       answerer = await answererPlugin.create(answererCfg);
@@ -569,7 +442,7 @@ export class Orchestrator {
         implementationHash,
         assertions: assertionImplementations,
         validatedAssertionsHash,
-        graderContext: this.harness,
+        graderContext: harness,
         skipJudge: request.skipJudge ?? false,
       });
       return await this.runWithPlugins(
@@ -581,16 +454,17 @@ export class Orchestrator {
           validatedAssertionConfigs,
           caseId,
           driverName,
+          permutationId: request.permutationId,
           answererName,
           answererConfig: answererCfg,
-          driverConfigHash: valueHash(driverCfg),
+          driverConfigHash: valueHash(execution ? { config: driverCfg, execution } : driverCfg),
           answererConfigHash: valueHash(answererCfg),
           validatedAssertionsHash,
           graderConfigHash: graderHash,
           evaluatorConfigHash: valueHash({
             luxVersion: LUX_VERSION,
             implementationHash,
-            harness: this.harness,
+            harness,
             driver: { name: driverPlugin.name, create: driverPlugin.create },
             answerer: { name: answererPlugin.name, create: answererPlugin.create },
             assertions: assertionImplementations,
@@ -599,6 +473,7 @@ export class Orchestrator {
             skipJudge: request.skipJudge ?? false,
           }),
           fixtureValues: valuesForFixtures(lifecycleFixtures),
+          environment: this.environment,
           driver,
           answerer,
         },
@@ -797,9 +672,12 @@ export class Orchestrator {
             observer: handle ?? observer,
             abortSignal: opts.abortSignal,
             suiteId: suite.manifest.suiteId,
+            suiteConcurrency: concurrency,
             subjectRoot: opts.subjectRoot,
             subjectHash: opts.subjectHash,
             profileId: opts.profileId,
+            driver: item.driver,
+            permutationId: item.permutationId,
             experimentId: opts.experimentId,
             variantId: opts.variantId,
             candidateId: opts.candidateId,