@prismatic-io/lux 0.0.1

package/src/orchestrator/experiment-finish.ts

@@ -0,0 +1,33 @@
+import type { z } from "zod";
+import type { Candidate, ExperimentCampaign } from "../core/index.js";
+import type { OpenedExperimentRun } from "./experiment-runs.js";
+import {
+  deriveExperimentSelection,
+  type ExperimentSelectionDerivationOptionsSchema,
+} from "./experiment-selection.js";
+
+export const finishExperiment = async (
+  experiment: OpenedExperimentRun,
+  campaign: ExperimentCampaign,
+  candidates: readonly Candidate[],
+  selectionOptions: z.input<typeof ExperimentSelectionDerivationOptionsSchema> = {},
+): Promise<Candidate | null> => {
+  const selection = deriveExperimentSelection(
+    campaign,
+    candidates,
+    experiment.manifest,
+    selectionOptions,
+  );
+  experiment.manifest.selection = selection;
+  let champion: Candidate | null = null;
+  if (selection.championCandidateId) {
+    champion =
+      candidates.find((candidate) => candidate.id === selection.championCandidateId) ?? null;
+  }
+  experiment.manifest.status = champion ? "passed" : "failed";
+  experiment.manifest.endedAt = new Date().toISOString();
+  if (champion) experiment.manifest.championCandidateId = champion.id;
+  else delete experiment.manifest.championCandidateId;
+  await experiment.write();
+  return champion;
+};

package/src/orchestrator/experiment-identity.ts

@@ -0,0 +1,356 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { bindSubjectRoot, contentHash, valueHash } from "../core/index.js";
+import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
+import { authoredDependencyGraphHash } from "./authored-dependencies.js";
+import { assertValidationArgumentsBound } from "./candidates.js";
+import { selectCases } from "./discover.js";
+import { evaluatorImplementationHash } from "./evaluator-identity.js";
+import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import { profileOrThrow } from "./experiment-context.js";
+import { caseRubricCliReferences, selectorsBySplit } from "./experiment-corpus.js";
+import {
+  claudeConfigRoot,
+  claudeUserBehaviorHash,
+  cliIdentity,
+  configuredClaudeSourceHashes,
+  LUX_VERSION,
+  type RuntimeCliReference,
+  RuntimeCliReferenceSchema,
+  referencedAnswererCli,
+  referencedCli,
+  runtimeCliIdentityHashForReferences,
+} from "./experiment-runtime-identity.js";
+import { loadEvalCase } from "./loader.js";
+import {
+  boundExperimentDriverConfig,
+  resolveAnswererReference,
+  resolveDriverReference,
+} from "./orchestrator.js";
+import { runtimeIdentity } from "./provenance.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+
+export const resolvedProfileSnapshot = async (
+  execution: CampaignExecutionOptions,
+): Promise<Record<string, unknown>> => {
+  const behaviorHashes = new Map<string, Promise<string | null>>();
+  const configuredSourceHashes = new Map<string, Promise<string>>();
+  const behaviorHash = (root: string): Promise<string | null> => {
+    const existing = behaviorHashes.get(root);
+    if (existing) return existing;
+    const created = claudeUserBehaviorHash(root);
+    behaviorHashes.set(root, created);
+    return created;
+  };
+  return Object.fromEntries(
+    await Promise.all(
+      execution.loaded.campaign.profiles.map(async (profileId) => {
+        const profile = profileOrThrow(execution.config, profileId);
+        // Offline grader campaigns never launch a subject driver. Keep their
+        // identity independent of irrelevant local agent settings/plugins
+        // while still snapshotting the harness that performs regrading.
+        const driver =
+          execution.loaded.campaign.evaluation.kind === "agent"
+            ? (profile.driver ?? execution.config.defaultDriver)
+            : undefined;
+        const driverCli = referencedCli(driver);
+        const harness = profile.harness ?? execution.config.harness;
+        const answerer =
+          execution.loaded.campaign.evaluation.kind === "agent"
+            ? (profile.answerer ?? execution.config.defaultAnswerer)
+            : undefined;
+        const answererCli = referencedAnswererCli(answerer);
+        const driverUserBehaviorHash =
+          driverCli?.name === "claude-code" ? await behaviorHash(claudeConfigRoot(driver)) : null;
+        const driverConfiguredSources = await configuredClaudeSourceHashes(
+          driver,
+          execution.config.rootDir,
+          configuredSourceHashes,
+        );
+        let harnessCommand = harness?.command;
+        if (!harnessCommand) {
+          harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
+        }
+        const capturesHarnessIdentity =
+          Boolean(harness) || execution.loaded.campaign.evaluation.kind === "grader";
+        const harnessCliIdentity = capturesHarnessIdentity
+          ? await cliIdentity(harnessCommand, execution.config.rootDir)
+          : null;
+        return [
+          profileId,
+          {
+            id: profileId,
+            driver: driver ?? null,
+            driverCliIdentity: driverCli
+              ? await cliIdentity(driverCli.command, execution.config.rootDir)
+              : null,
+            ...(driverCli?.name === "claude-code"
+              ? {
+                  driverClaudeConfigRoot: claudeConfigRoot(driver),
+                  driverUserBehaviorHash,
+                  driverConfiguredSources,
+                }
+              : {}),
+            answerer,
+            answererCliIdentity: answererCli
+              ? await cliIdentity(answererCli.command, execution.config.rootDir)
+              : null,
+            harness: harness ?? null,
+            harnessCliIdentity,
+          },
+        ];
+      }),
+    ),
+  );
+};
+
+const trustedExecutableEntrypoints = async (
+  execution: CampaignExecutionOptions,
+  corpus: EvaluationCorpus,
+): Promise<Set<string>> => {
+  const paths = new Set<string>();
+  const add = (command: string | undefined): void => {
+    if (!command || !isAbsolute(command)) return;
+    const fromRoot = relative(execution.config.rootDir, resolve(command));
+    if (
+      fromRoot !== "" &&
+      fromRoot !== ".." &&
+      !fromRoot.startsWith(`..${sep}`) &&
+      !isAbsolute(fromRoot)
+    ) {
+      paths.add(fromRoot);
+    }
+  };
+  for (const validation of execution.loaded.campaign.subject.validate) add(validation.command);
+  if (execution.loaded.campaign.optimizer) {
+    const optimizer = ReflectiveOptimizerConfigSchema.parse(
+      execution.loaded.campaign.optimizer.config ?? {},
+    );
+    add(optimizer.command);
+  }
+  for (const profileId of execution.loaded.campaign.profiles) {
+    const profile = profileOrThrow(execution.config, profileId);
+    add(referencedCli(profile.driver ?? execution.config.defaultDriver)?.command);
+    add(referencedAnswererCli(profile.answerer ?? execution.config.defaultAnswerer)?.command);
+    add((profile.harness ?? execution.config.harness)?.command);
+  }
+  if (corpus.kind === "agent") {
+    const selected = [
+      ...new Map(
+        selectorsBySplit(execution.loaded.campaign)
+          .flatMap((split) => selectCases(corpus.cases, split.selector))
+          .map((evalCase) => [evalCase.path, evalCase]),
+      ).values(),
+    ];
+    for (const evalCase of selected) {
+      const authored = bindSubjectRoot(
+        await loadEvalCase(evalCase.path),
+        execution.loaded.subjectRoot,
+      );
+      add(referencedCli(authored.driver)?.command);
+      add(referencedAnswererCli(authored.answerer)?.command);
+      for (const reference of caseRubricCliReferences(execution, authored)) {
+        add(reference.command);
+      }
+    }
+  }
+  return paths;
+};
+
+export const runtimeCliReferences = async (
+  execution: CampaignExecutionOptions,
+  corpus: EvaluationCorpus,
+  mode: "experiment" | "optimize",
+): Promise<RuntimeCliReference[]> => {
+  const references = new Map<string, RuntimeCliReference>();
+  const add = (command: string | undefined, baseRoot: string): void => {
+    if (!command) return;
+    const reference = RuntimeCliReferenceSchema.parse({ command, baseRoot: resolve(baseRoot) });
+    references.set(`${reference.command}\u0000${reference.baseRoot}`, reference);
+  };
+
+  for (const validation of execution.loaded.campaign.subject.validate) {
+    add(validation.command, execution.loaded.subjectRoot);
+  }
+  if (mode === "optimize" && execution.loaded.campaign.optimizer) {
+    const optimizer = ReflectiveOptimizerConfigSchema.parse(
+      execution.loaded.campaign.optimizer.config ?? {},
+    );
+    add(
+      optimizer.command ?? (optimizer.provider === "codex" ? "codex" : "claude"),
+      execution.loaded.directory,
+    );
+  }
+  if (corpus.kind === "grader") {
+    for (const profileId of execution.loaded.campaign.profiles) {
+      const profile = profileOrThrow(execution.config, profileId);
+      const harness = profile.harness ?? execution.config.harness;
+      add(
+        harness?.command ?? (harness?.provider === "codex" ? "codex" : "claude"),
+        execution.config.rootDir,
+      );
+    }
+  }
+  if (corpus.kind === "agent") {
+    const selected = [
+      ...new Map(
+        selectorsBySplit(execution.loaded.campaign)
+          .flatMap((split) => selectCases(corpus.cases, split.selector))
+          .map((evalCase) => [evalCase.path, evalCase]),
+      ).values(),
+    ];
+    for (const evalCase of selected) {
+      const authored = bindSubjectRoot(
+        await loadEvalCase(evalCase.path),
+        execution.loaded.subjectRoot,
+      );
+      for (const profileId of execution.loaded.campaign.profiles) {
+        const profile = profileOrThrow(execution.config, profileId);
+        const harness = profile.harness ?? execution.config.harness;
+        const driver = resolveDriverReference(
+          authored,
+          profile.driver ?? execution.config.defaultDriver,
+        );
+        const answerer = resolveAnswererReference(
+          authored,
+          profile.answerer ?? execution.config.defaultAnswerer,
+          harness,
+        );
+        add(referencedCli(driver)?.command, execution.config.rootDir);
+        add(referencedAnswererCli(answerer)?.command, execution.config.rootDir);
+      }
+      for (const reference of caseRubricCliReferences(execution, authored)) {
+        add(reference.command, execution.config.rootDir);
+      }
+    }
+  }
+  return [...references.values()].sort((left, right) => {
+    const commandOrder = left.command.localeCompare(right.command);
+    return commandOrder || left.baseRoot.localeCompare(right.baseRoot);
+  });
+};
+
+const runtimeCliIdentityHash = async (
+  execution: CampaignExecutionOptions,
+  corpus: EvaluationCorpus,
+  mode: "experiment" | "optimize",
+): Promise<string> =>
+  runtimeCliIdentityHashForReferences(await runtimeCliReferences(execution, corpus, mode));
+
+const authoredConfigDependencyHash = async (
+  execution: CampaignExecutionOptions,
+  corpus: EvaluationCorpus,
+): Promise<string> => {
+  const configRoot = resolve(execution.config.rootDir);
+  const runsRoot = resolve(execution.runsRoot ?? execution.config.runsRoot);
+  const runsFromConfig = relative(configRoot, runsRoot);
+  if (runsFromConfig === "") {
+    throw new Error("experiment runsRoot cannot be the Lux config root");
+  }
+  const entrypoints = await trustedExecutableEntrypoints(execution, corpus);
+  const add = (path: string | null | undefined): void => {
+    if (path) entrypoints.add(relative(configRoot, resolve(path)));
+  };
+  add(execution.config.configPath);
+  add(execution.loaded.path);
+  if (corpus.kind === "agent") {
+    for (const split of selectorsBySplit(execution.loaded.campaign)) {
+      for (const evalCase of selectCases(corpus.cases, split.selector)) add(evalCase.path);
+    }
+  } else {
+    add(execution.loaded.graderAnnotations?.path);
+  }
+  return authoredDependencyGraphHash(configRoot, [...entrypoints]);
+};
+const validateAgentCaseProfileDrivers = async (
+  execution: CampaignExecutionOptions,
+  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+): Promise<void> => {
+  const selected = [
+    ...new Map(
+      selectorsBySplit(execution.loaded.campaign)
+        .flatMap((split) => selectCases(corpus.cases, split.selector))
+        .map((evalCase) => [evalCase.path, evalCase]),
+    ).values(),
+  ];
+  for (const evalCase of selected) {
+    const authored = bindSubjectRoot(
+      await loadEvalCase(evalCase.path),
+      execution.loaded.subjectRoot,
+    );
+    for (const profileId of execution.loaded.campaign.profiles) {
+      const profile = profileOrThrow(execution.config, profileId);
+      const reference = resolveDriverReference(
+        authored,
+        profile.driver ?? execution.config.defaultDriver,
+      );
+      const name = typeof reference === "string" ? reference : reference.name;
+      const rawConfig = typeof reference === "object" ? reference.config : undefined;
+      boundExperimentDriverConfig(name, rawConfig, "identity-check");
+      const plugin = execution.registry.getDriver(name);
+      if (!plugin) {
+        throw new Error(
+          `case '${evalCase.id}' profile '${profileId}' references unknown driver '${name}'`,
+        );
+      }
+      const validated = validatePluginConfig(
+        plugin.configSchema,
+        rawConfig,
+        `case '${evalCase.id}' profile '${profileId}' driver '${name}'`,
+      );
+      boundExperimentDriverConfig(name, validated, "identity-check");
+    }
+  }
+};
+
+export const evaluationIdentity = async (
+  execution: CampaignExecutionOptions,
+  corpus: EvaluationCorpus,
+  mode: "experiment" | "optimize",
+  boundRuntimeCliIdentityHash?: string,
+): Promise<string> => {
+  const profiles = await resolvedProfileSnapshot(execution);
+  if (corpus.kind === "agent") {
+    await validateAgentCaseProfileDrivers(execution, corpus);
+  }
+  const configSourceHash = execution.config.configPath
+    ? contentHash(await readFile(execution.config.configPath))
+    : null;
+  const configDependencyHash = await authoredConfigDependencyHash(execution, corpus);
+  const optimizerConfig =
+    mode === "optimize" && execution.loaded.campaign.optimizer
+      ? ReflectiveOptimizerConfigSchema.parse(execution.loaded.campaign.optimizer.config ?? {})
+      : undefined;
+  let optimizerCommand: string | undefined;
+  if (optimizerConfig) {
+    optimizerCommand = optimizerConfig.command;
+    if (!optimizerCommand) {
+      optimizerCommand = optimizerConfig.provider === "codex" ? "codex" : "claude";
+    }
+  }
+  return valueHash({
+    evaluationKind: execution.loaded.campaign.evaluation.kind,
+    evaluatorVersion: 1,
+    luxVersion: LUX_VERSION,
+    evaluatorImplementationHash: await evaluatorImplementationHash(),
+    runtime: runtimeIdentity(),
+    corpusHash: corpus.hash,
+    profiles,
+    configSourceHash,
+    configDependencyHash,
+    validationCliIdentities: await Promise.all(
+      execution.loaded.campaign.subject.validate.map(async (validation) => {
+        assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+        return {
+          args: validation.args,
+          cli: await cliIdentity(validation.command, execution.loaded.subjectRoot),
+        };
+      }),
+    ),
+    optimizerCliIdentity: optimizerCommand
+      ? await cliIdentity(optimizerCommand, execution.loaded.directory)
+      : null,
+    runtimeCliIdentityHash:
+      boundRuntimeCliIdentityHash ?? (await runtimeCliIdentityHash(execution, corpus, mode)),
+  });
+};

package/src/orchestrator/experiment-loader.ts

@@ -0,0 +1,105 @@
+import { lstat, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { z } from "zod";
+import {
+  type ExperimentCampaign,
+  ExperimentCampaignSchema,
+  formatZodIssues,
+  valueHash,
+} from "../core/index.js";
+import { LoadedGraderAnnotationsSchema, loadGraderAnnotations } from "./annotation-loader.js";
+
+export const LoadedExperimentCampaignSchema = z
+  .object({
+    path: z.string(),
+    directory: z.string(),
+    campaign: ExperimentCampaignSchema,
+    subjectRoot: z.string(),
+    campaignHash: z.string(),
+    graderAnnotations: LoadedGraderAnnotationsSchema.optional(),
+  })
+  .strict();
+export type LoadedExperimentCampaign = z.infer<typeof LoadedExperimentCampaignSchema>;
+
+export const ExperimentCommandModeSchema = z.enum(["experiment", "optimize"]);
+export type ExperimentCommandMode = z.infer<typeof ExperimentCommandModeSchema>;
+
+export const experimentCampaignMode = (
+  path: string,
+  campaign: ExperimentCampaign,
+  requestedMode?: ExperimentCommandMode,
+): ExperimentCommandMode => {
+  const experimentSuffix = path.endsWith(".experiment.ts");
+  const optimizeSuffix = path.endsWith(".optimize.ts");
+  if (experimentSuffix && campaign.optimizer) {
+    throw new Error(
+      "a .experiment.ts campaign must not define an optimizer; rename it to .optimize.ts or remove the optimizer",
+    );
+  }
+  if (optimizeSuffix && !campaign.optimizer) {
+    throw new Error(
+      "a .optimize.ts campaign must define an optimizer; rename it to .experiment.ts or add one",
+    );
+  }
+  let authoredMode: ExperimentCommandMode = "experiment";
+  if (campaign.optimizer) authoredMode = "optimize";
+  if (requestedMode && requestedMode !== authoredMode) {
+    throw new Error(
+      `lux ${requestedMode} cannot run this ${authoredMode} campaign; use \`lux ${authoredMode}\` or change its optimizer`,
+    );
+  }
+  return authoredMode;
+};
+
+export const resolveExperimentCampaignPath = async (
+  projectRoot: string,
+  filePath: string,
+): Promise<string> => {
+  const root = resolve(projectRoot);
+  const path = resolve(root, filePath);
+  const fromRoot = relative(root, path);
+  if (isAbsolute(fromRoot) || fromRoot === ".." || fromRoot.startsWith(`..${sep}`)) {
+    throw new Error(`experiment campaign must be contained by the Lux project: ${path}`);
+  }
+  const info = await lstat(path);
+  if (info.isSymbolicLink() || !info.isFile()) {
+    throw new Error(`experiment campaign must be a regular, non-symlink file: ${path}`);
+  }
+  const [canonicalRoot, canonicalPath] = await Promise.all([realpath(root), realpath(path)]);
+  if (canonicalPath !== resolve(canonicalRoot, fromRoot)) {
+    throw new Error(`experiment campaign path must not contain symlinks: ${path}`);
+  }
+  return canonicalPath;
+};
+
+export const loadExperimentCampaign = async (
+  filePath: string,
+): Promise<LoadedExperimentCampaign> => {
+  const path = resolve(filePath);
+  const mod = (await import(pathToFileURL(path).href)) as { default?: unknown };
+  if (mod.default === undefined) {
+    throw new Error(
+      `Lux experiment at ${path} has no default export. Use \`export default defineExperiment({ ... })\`.`,
+    );
+  }
+  const parsed = ExperimentCampaignSchema.safeParse(mod.default);
+  if (!parsed.success) {
+    throw new Error(`Invalid Lux experiment at ${path}: ${formatZodIssues(parsed.error)}`);
+  }
+  const directory = dirname(path);
+  const graderAnnotations =
+    parsed.data.evaluation.kind === "grader"
+      ? await loadGraderAnnotations(resolve(directory, parsed.data.evaluation.annotations))
+      : undefined;
+  return LoadedExperimentCampaignSchema.parse({
+    path,
+    directory,
+    campaign: parsed.data,
+    subjectRoot: resolve(directory, parsed.data.subject.root),
+    campaignHash: graderAnnotations
+      ? valueHash({ campaign: parsed.data, annotationsHash: graderAnnotations.hash })
+      : valueHash(parsed.data),
+    ...(graderAnnotations ? { graderAnnotations } : {}),
+  });
+};