@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/doctor.ts

@@ -4,18 +4,39 @@ import { resolve } from "node:path";
 import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
-import type { EvalCase, Registry } from "../core/index.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
+import {
+  casePermutations,
+  type DriverRef,
+  type EvalCase,
+  expandHarnessMatrix,
+  type HarnessPermutation,
+  permutationDriverRef,
+  permutationId,
+  type Registry,
+} from "../core/index.js";
+import { mergeDriverReference, withDriverDefaults } from "../core/reference-defaults.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
-import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
+import { validateDriverCapabilities } from "./driver-capabilities.js";
+import { ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
 import {
   experimentCampaignMode,
   loadExperimentCampaign,
   resolveExperimentCampaignPath,
 } from "./experiment-loader.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
 import { assertionConfigForSchema } from "./grading.js";
+import {
+  describeHarnessCatalogDrift,
+  harnessCatalogDrift,
+  harnessCatalogProbes,
+  hasHarnessCatalogDrift,
+} from "./harness-catalog-drift.js";
 import { loadEvalCase } from "./loader.js";
+import { resolveAnswererReference } from "./orchestrator.js";
+import { resolveProfile } from "./profile.js";
 import { validatePluginConfig, validatePluginConfigAsync } from "./validate-plugin-config.js";
 
 export const DoctorDiagnosticSchema = z
@@ -60,6 +81,7 @@ export const DoctorOptionsSchema = z
   .object({
     cwd: z.string().min(1),
     campaigns: z.array(z.string()).optional(),
+    static: z.boolean().optional(),
   })
   .strict();
 export type DoctorOptions = z.input<typeof DoctorOptionsSchema>;
@@ -102,57 +124,61 @@ const rawRefConfig = (reference: unknown): unknown =>
     ? (reference as { config?: unknown }).config
     : undefined;
 
-const mergeRecords = (base: unknown, local: unknown): unknown => {
-  if (
-    !base ||
-    typeof base !== "object" ||
-    Array.isArray(base) ||
-    !local ||
-    typeof local !== "object" ||
-    Array.isArray(local)
-  ) {
-    return local ?? base;
-  }
-  const merged: Record<string, unknown> = { ...(base as Record<string, unknown>) };
-  for (const [key, value] of Object.entries(local as Record<string, unknown>)) {
-    merged[key] = key in merged ? mergeRecords(merged[key], value) : value;
-  }
-  return merged;
-};
-
-const effectiveDriverReference = (local: unknown, fallback: unknown): unknown => {
-  if (!local) return fallback;
-  if (!fallback || refName(local) !== refName(fallback)) return local;
-  return {
-    name: refName(local),
-    config: mergeRecords(rawRefConfig(fallback), rawRefConfig(local)),
-  };
+const boundPermutation = (
+  permutation: HarnessPermutation,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): DriverRef => {
+  const plugin = registry.getDriver(permutation.driver);
+  if (!plugin) throw new Error(`unknown driver '${permutation.driver}'`);
+  return withDriverDefaults(
+    permutationDriverRef(permutation, config.defaultDriver, plugin),
+    config.driverDefaults,
+  ) as DriverRef;
 };
 
-const caseDriverReferences = (local: unknown, config: ResolvedLuxConfig): unknown[] => {
+const caseDriverReferences = (
+  authored: Pick<EvalCase, "driver" | "matrix">,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): unknown[] => {
+  const local = authored.driver;
+  const permutations = casePermutations(authored, config.matrix);
+  if (permutations.length > 0) {
+    return permutations.map((permutation) =>
+      mergeDriverReference(boundPermutation(permutation, config, registry), local),
+    );
+  }
   const fallbacks = [
-    config.defaultDriver,
-    ...Object.values(config.profiles ?? {}).map((profile) => profile.driver),
+    resolveProfile(config).driver,
+    ...Object.values(config.profiles ?? {}).map(
+      (profile) => resolveProfile(config, profile).driver,
+    ),
   ].filter((reference) => reference !== undefined);
   if (!local) return [...new Set(fallbacks)];
-  if (fallbacks.length === 0) return [local];
-  const references = fallbacks.map((fallback) => effectiveDriverReference(local, fallback));
+  if (fallbacks.length === 0) return [withDriverDefaults(local, config.driverDefaults)];
+  const references = fallbacks.map((fallback) =>
+    withDriverDefaults(mergeDriverReference(fallback, local), config.driverDefaults),
+  );
   return [
     ...new Map(references.map((reference) => [JSON.stringify(reference), reference])).values(),
   ];
 };
 
-const driverCommand = (reference: unknown): string | undefined => {
+const driverCommands = (reference: unknown, registry: Registry): string[] => {
   const name = refName(reference);
-  const config = refConfig(reference);
-  if (name === "claude-code") {
-    return typeof config.command === "string" ? config.command : "claude";
-  }
-  if (name === "codex") return typeof config.command === "string" ? config.command : "codex";
-  if (name === "subprocess") {
-    return typeof config.command === "string" ? config.command : undefined;
+  const plugin = name ? registry.getDriver(name) : undefined;
+  if (!plugin?.runtime) return [];
+  try {
+    const config = validatePluginConfig(
+      plugin.configSchema,
+      rawRefConfig(reference),
+      `driver '${name}'`,
+    );
+    return plugin.runtime(config).commands;
+  } catch {
+    return [];
   }
-  return undefined;
 };
 
 const answererCommand = (reference: unknown): string | undefined => {
@@ -167,7 +193,9 @@ const rubricCommands = (assertion: unknown, config: ResolvedLuxConfig): string[]
   const rubric = RubricConfigSchema.parse(assertion);
   const harnesses = [
     config.harness,
-    ...Object.values(config.profiles ?? {}).map((profile) => profile.harness ?? config.harness),
+    ...Object.values(config.profiles ?? {}).map(
+      (profile) => resolveProfile(config, profile).harness,
+    ),
   ];
   const commands = harnesses.map((harness) => {
     const provider = rubric.provider ?? harness?.provider ?? "claude-code";
@@ -179,12 +207,24 @@ const rubricCommands = (assertion: unknown, config: ResolvedLuxConfig): string[]
   return [...new Set(commands)];
 };
 
-const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
+const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): CliTarget[] => {
   const targets: CliTarget[] = [];
   const add = (label: string, command: string | undefined): void => {
     if (command) targets.push({ label, command });
   };
-  add("default driver", driverCommand(config.defaultDriver));
+  for (const command of driverCommands(resolveProfile(config).driver, registry))
+    add("default driver", command);
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    try {
+      for (const command of driverCommands(
+        boundPermutation(permutation, config, registry),
+        registry,
+      ))
+        add(`matrix '${permutationId(permutation)}' driver`, command);
+    } catch {
+      /* Reported by validateConfiguredMatrix. */
+    }
+  }
   add("default answerer", answererCommand(config.defaultAnswerer));
   if (config.harness) {
     add(
@@ -195,9 +235,11 @@ const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
   for (const [id, profile] of Object.entries(config.profiles ?? {}).sort(([a], [b]) =>
     a.localeCompare(b),
   )) {
-    add(`profile '${id}' driver`, driverCommand(profile.driver ?? config.defaultDriver));
-    add(`profile '${id}' answerer`, answererCommand(profile.answerer ?? config.defaultAnswerer));
-    const harness = profile.harness ?? config.harness;
+    for (const command of driverCommands(resolveProfile(config, profile).driver, registry)) {
+      add(`profile '${id}' driver`, command);
+    }
+    add(`profile '${id}' answerer`, answererCommand(resolveProfile(config, profile).answerer));
+    const harness = resolveProfile(config, profile).harness;
     if (harness) {
       add(
         `profile '${id}' harness`,
@@ -299,18 +341,18 @@ const loadDoctorRegistry = (
 const configuredDriverReferences = (
   config: ResolvedLuxConfig,
 ): Array<readonly [string, unknown]> => [
-  ["default", config.defaultDriver],
+  ["default", resolveProfile(config).driver],
   ...Object.entries(config.profiles ?? {}).map(
-    ([id, profile]) => [`profile '${id}'`, profile.driver] as const,
+    ([id, profile]) => [`profile '${id}'`, resolveProfile(config, profile).driver] as const,
   ),
 ];
 
 const configuredAnswererReferences = (
   config: ResolvedLuxConfig,
 ): Array<readonly [string, unknown]> => [
-  ["default", config.defaultAnswerer],
+  ["default", resolveProfile(config).answerer],
   ...Object.entries(config.profiles ?? {}).map(
-    ([id, profile]) => [`profile '${id}'`, profile.answerer] as const,
+    ([id, profile]) => [`profile '${id}'`, resolveProfile(config, profile).answerer] as const,
   ),
 ];
 
@@ -348,6 +390,32 @@ const validateConfiguredDrivers = (
   }
 };
 
+const validateConfiguredMatrix = (
+  config: ResolvedLuxConfig,
+  registry: Registry,
+  recorder: DoctorRecorder,
+): void => {
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    const id = permutationId(permutation);
+    try {
+      const reference = boundPermutation(permutation, config, registry);
+      const plugin = registry.getDriver(permutation.driver);
+      validatePluginConfig(
+        plugin?.configSchema,
+        rawRefConfig(reference),
+        `matrix '${id}' driver '${permutation.driver}'`,
+      );
+    } catch (error) {
+      recorder.add(
+        `matrix:${id}`,
+        "fail",
+        `matrix permutation '${id}' is not runnable`,
+        firstLine(error),
+      );
+    }
+  }
+};
+
 const validateConfiguredAnswerers = (
   config: ResolvedLuxConfig,
   registry: Registry,
@@ -418,17 +486,19 @@ const caseDriverFailure = (
   caseId: string,
   reference: unknown,
   registry: Registry,
+  required: EvalCase["requires"],
 ): string | null => {
   const name = refName(reference);
   if (!name) return "driver reference has no name";
   const plugin = registry.getDriver(name);
   if (!plugin) return `unknown driver '${name}'`;
   try {
-    validatePluginConfig(
+    const config = validatePluginConfig(
       plugin.configSchema,
       rawRefConfig(reference),
       `case '${caseId}' driver '${name}'`,
     );
+    validateDriverCapabilities(plugin, config, required, driverExecutionOf(reference));
     return null;
   } catch (error) {
     return firstLine(error);
@@ -440,6 +510,7 @@ const validateCaseDrivers = (
   drivers: readonly unknown[],
   registry: Registry,
   recorder: DoctorRecorder,
+  required: EvalCase["requires"],
 ): void => {
   if (drivers.length === 0) {
     recorder.add(
@@ -450,7 +521,7 @@ const validateCaseDrivers = (
     return;
   }
   const failures = drivers
-    .map((driver) => caseDriverFailure(caseId, driver, registry))
+    .map((driver) => caseDriverFailure(caseId, driver, registry, required))
     .filter((failure): failure is string => failure !== null);
   if (failures.length === drivers.length) {
     recorder.add(
@@ -542,13 +613,16 @@ const executionTargetsForCase = (
   caseId: string,
   drivers: readonly unknown[],
   answerer: unknown,
+  registry: Registry,
 ): CliTarget[] => {
   const targets = drivers.flatMap((driver): CliTarget[] => {
-    const command = driverCommand(driver);
-    return command ? [{ label: `case '${caseId}' driver`, command }] : [];
+    return driverCommands(driver, registry).map((command) => ({
+      label: `case '${caseId}' driver`,
+      command,
+    }));
   });
-  const command = answererCommand(answerer);
-  if (command) targets.push({ label: `case '${caseId}' answerer`, command });
+  const answererCli = answererCommand(answerer);
+  if (answererCli) targets.push({ label: `case '${caseId}' answerer`, command: answererCli });
   return targets;
 };
 
@@ -561,13 +635,13 @@ const inspectCase = async (
   try {
     const authored = await loadEvalCase(discovered.path);
     const caseRegistry = registryForEvalCase(registry, authored);
-    const drivers = caseDriverReferences(authored.driver, config);
-    const answerer = authored.answerer ?? config.defaultAnswerer;
-    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
+    const drivers = caseDriverReferences(authored, config, caseRegistry);
+    const answerer = resolveAnswererReference(authored, config.defaultAnswerer, config.harness);
+    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder, authored.requires);
     validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [
       ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
-      ...executionTargetsForCase(discovered.id, drivers, answerer),
+      ...executionTargetsForCase(discovered.id, drivers, answerer, caseRegistry),
     ];
   } catch (error) {
     recorder.add(
@@ -644,6 +718,54 @@ const inspectCliTargets = async (
   }
 };
 
+const catalogDriverReferences = (config: ResolvedLuxConfig): unknown[] => [
+  ...configuredDriverReferences(config).map(([, reference]) => reference),
+  ...(config.matrix ?? []).map((entry) => ({ name: entry.driver, config: entry.config })),
+];
+
+const configuredDriverCommand = (config: ResolvedLuxConfig, driver: string): string | undefined => {
+  for (const reference of catalogDriverReferences(config)) {
+    if (refName(reference) !== driver) continue;
+    const command = (rawRefConfig(reference) as { command?: unknown } | undefined)?.command;
+    if (typeof command === "string") return command;
+  }
+  return undefined;
+};
+
+const inspectHarnessCatalogs = async (
+  config: ResolvedLuxConfig,
+  recorder: DoctorRecorder,
+): Promise<void> => {
+  const configured = new Set(
+    catalogDriverReferences(config)
+      .map(refName)
+      .filter((name): name is string => name !== undefined),
+  );
+  for (const probe of harnessCatalogProbes) {
+    const driver = probe.catalog.driver;
+    if (!configured.has(driver)) continue;
+    const live = await probe.read(configuredDriverCommand(config, driver));
+    if (!live) {
+      recorder.add(
+        `catalog:${driver}`,
+        "warn",
+        `${driver} harness catalog (verified ${probe.catalog.verified}) could not be compared with the installed CLI`,
+        `${probe.source} was unavailable`,
+      );
+      continue;
+    }
+    const drift = harnessCatalogDrift(probe.catalog, live);
+    recorder.add(
+      `catalog:${driver}`,
+      hasHarnessCatalogDrift(drift) ? "warn" : "pass",
+      hasHarnessCatalogDrift(drift)
+        ? `${driver} harness catalog (verified ${probe.catalog.verified}) differs from the installed CLI`
+        : `${driver} harness catalog matches the installed CLI`,
+      describeHarnessCatalogDrift(drift) || undefined,
+    );
+  }
+};
+
 const isBlockingCampaignWarning = (warning: string): boolean =>
   warning.includes("below confidence minSamples=") || warning.includes("cannot attain confidence");
 
@@ -652,6 +774,7 @@ const inspectCampaign = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   try {
     const loaded = await loadExperimentCampaign(path);
@@ -665,7 +788,9 @@ const inspectCampaign = async (
         placeholders.join(", "),
       );
     }
-    const plan = await planExperimentCampaign({ config, registry, loaded }, mode);
+    const plan = await planExperimentCampaign({ config, registry, loaded }, mode, {
+      static: staticChecks,
+    });
     recorder.campaigns.push({ path, campaignId: loaded.campaign.id, mode, plan });
     recorder.add(
       `campaign:${loaded.campaign.id}`,
@@ -697,6 +822,7 @@ const inspectCampaigns = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   let paths: string[];
   try {
@@ -719,7 +845,7 @@ const inspectCampaigns = async (
     );
   }
   for (const path of paths) {
-    await inspectCampaign(path, config, registry, recorder);
+    await inspectCampaign(path, config, registry, recorder, staticChecks);
   }
 };
 
@@ -733,10 +859,30 @@ export const doctorProject = async (options: DoctorOptions): Promise<DoctorRepor
   if (!registry) return recorder.finish(config.rootDir);
 
   validateConfiguredDrivers(config, registry, recorder);
+  validateConfiguredMatrix(config, registry, recorder);
   validateConfiguredAnswerers(config, registry, recorder);
   const caseCliTargets = await inspectCases(config, registry, recorder);
-  await inspectCliTargets([...configuredCliTargets(config), ...caseCliTargets], config, recorder);
-  await inspectCampaigns(parsedOptions.campaigns ?? [], config, registry, recorder);
+  if (parsedOptions.static) {
+    recorder.add(
+      "static-checks",
+      "warn",
+      "static checks only; installed executables and campaign execution identity were not checked",
+    );
+  } else {
+    await inspectCliTargets(
+      [...configuredCliTargets(config, registry), ...caseCliTargets],
+      config,
+      recorder,
+    );
+    await inspectHarnessCatalogs(config, recorder);
+  }
+  await inspectCampaigns(
+    parsedOptions.campaigns ?? [],
+    config,
+    registry,
+    recorder,
+    parsedOptions.static ?? false,
+  );
   return recorder.finish(config.rootDir);
 };
 

package/src/orchestrator/driver-capabilities.ts

@@ -0,0 +1,21 @@
+import type { DriverCapability, DriverPlugin } from "../core/driver.js";
+import type { DriverExecution } from "../core/driver-execution.js";
+
+export const validateDriverCapabilities = (
+  plugin: DriverPlugin,
+  config: unknown,
+  required: readonly DriverCapability[] = [],
+  execution?: DriverExecution,
+): void => {
+  if (required.length === 0) return;
+  const capabilities = {
+    ...plugin.runtime?.(config).capabilities,
+    ...(execution?.kind === "container" ? { isolation: true } : {}),
+  };
+  const missing = required.filter((capability) => capabilities?.[capability] !== true);
+  if (missing.length > 0) {
+    throw new Error(
+      `driver '${plugin.name}' does not declare required capabilities: ${missing.join(", ")}`,
+    );
+  }
+};

package/src/orchestrator/driver-identity.ts

@@ -0,0 +1,69 @@
+import type { DriverPlugin, DriverRuntime } from "../core/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
+import type { Registry } from "../core/registry.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+
+const resolvedDriver = (
+  reference: unknown,
+  registry: Registry,
+): { plugin: DriverPlugin; config: unknown } | undefined => {
+  const name =
+    typeof reference === "string" ? reference : (reference as { name?: unknown } | null)?.name;
+  if (typeof name !== "string") return undefined;
+  const plugin = registry.getDriver(name);
+  if (!plugin) throw new Error(`unknown driver '${name}'`);
+  const raw =
+    typeof reference === "object" && reference !== null
+      ? (reference as { config?: unknown }).config
+      : undefined;
+  return { plugin, config: validatePluginConfig(plugin.configSchema, raw, `driver '${name}'`) };
+};
+
+export const referencedDriverRuntime = (reference: unknown, registry: Registry): DriverRuntime => {
+  const resolved = resolvedDriver(reference, registry);
+  const runtime = resolved?.plugin.runtime?.(resolved.config) ?? { commands: [], capabilities: {} };
+  const execution = driverExecutionOf(reference);
+  return execution?.kind === "container"
+    ? {
+        commands: [execution.command ?? "docker"],
+        capabilities: { ...runtime.capabilities, isolation: true },
+      }
+    : runtime;
+};
+
+export const experimentDriverIdentity = async (
+  reference: unknown,
+  registry: Registry,
+  rootDir: string,
+): Promise<unknown> => {
+  if (driverExecutionOf(reference)?.kind === "container")
+    throw new Error("container execution does not yet support controlled experiments");
+  const resolved = resolvedDriver(reference, registry);
+  if (!resolved) return null;
+  const { plugin, config } = resolved;
+  const experiment = plugin.experiment;
+  if (!experiment)
+    throw new Error(
+      `driver '${plugin.name}' supports ordinary eval runs but does not declare controlled experiment support`,
+    );
+  if (!experiment.version.trim())
+    throw new Error(`driver '${plugin.name}' experiment version must not be empty`);
+  const prepared = validatePluginConfig(
+    plugin.configSchema,
+    experiment.prepare(config),
+    `driver '${plugin.name}' experiment configuration`,
+  );
+  const runtime = plugin.runtime?.(prepared) ?? { commands: [], capabilities: {} };
+  const behavior = await experiment.identity(config, { rootDir });
+  return {
+    driver: plugin.name,
+    version: experiment.version,
+    config: prepared,
+    capabilities: runtime.capabilities,
+    executables: await Promise.all(
+      runtime.commands.map((command) => cliIdentity(command, rootDir)),
+    ),
+    behavior,
+  };
+};

package/src/orchestrator/experiment-context.ts

@@ -2,18 +2,20 @@ import type { Registry } from "../core/index.js";
 import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
+import { resolveProfile } from "./profile.js";
 
-export type EvaluationCorpus =
+export type CorpusSelection =
   | {
       kind: "agent";
       cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
-      hash: string;
     }
   | {
       kind: "grader";
       hash: string;
     };
 
+export type EvaluationCorpus = CorpusSelection & { hash: string };
+
 export type CampaignExecutionOptions = {
   config: ResolvedLuxConfig;
   registry: Registry;
@@ -37,5 +39,5 @@ export const profileOrThrow = (config: ResolvedLuxConfig, profileId: string) =>
       `experiment profile '${profileId}' is not configured${available.length > 0 ? `; available: ${available.join(", ")}` : ""}`,
     );
   }
-  return profile;
+  return resolveProfile(config, profile);
 };