@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/experiment-identity.ts

@@ -1,24 +1,23 @@
 import { readFile } from "node:fs/promises";
 import { isAbsolute, relative, resolve, sep } from "node:path";
 import { bindSubjectRoot, contentHash, valueHash } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
 import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
 import { authoredDependencyGraphHash } from "./authored-dependencies.js";
 import { assertValidationArgumentsBound } from "./candidates.js";
 import { selectCases } from "./discover.js";
+import { experimentDriverIdentity, referencedDriverRuntime } from "./driver-identity.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { caseRubricCliReferences, selectorsBySplit } from "./experiment-corpus.js";
 import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
   cliIdentity,
-  configuredClaudeSourceHashes,
   LUX_VERSION,
   type RuntimeCliReference,
   RuntimeCliReferenceSchema,
   referencedAnswererCli,
-  referencedCli,
   runtimeCliIdentityHashForReferences,
 } from "./experiment-runtime-identity.js";
 import { loadEvalCase } from "./loader.js";
@@ -28,20 +27,13 @@ import {
   resolveDriverReference,
 } from "./orchestrator.js";
 import { runtimeIdentity } from "./provenance.js";
+import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
+import { loadRubricDefinition } from "./rubric-loader.js";
 import { validatePluginConfig } from "./validate-plugin-config.js";
 
 export const resolvedProfileSnapshot = async (
   execution: CampaignExecutionOptions,
 ): Promise<Record<string, unknown>> => {
-  const behaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
-  const behaviorHash = (root: string): Promise<string | null> => {
-    const existing = behaviorHashes.get(root);
-    if (existing) return existing;
-    const created = claudeUserBehaviorHash(root);
-    behaviorHashes.set(root, created);
-    return created;
-  };
   return Object.fromEntries(
     await Promise.all(
       execution.loaded.campaign.profiles.map(async (profileId) => {
@@ -53,26 +45,24 @@ export const resolvedProfileSnapshot = async (
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.driver ?? execution.config.defaultDriver)
             : undefined;
-        const driverCli = referencedCli(driver);
-        const harness = profile.harness ?? execution.config.harness;
+        const harness = resolveHarnessForDriver(
+          profile.harness ?? execution.config.harness,
+          driver,
+        );
         const answerer =
           execution.loaded.campaign.evaluation.kind === "agent"
-            ? (profile.answerer ?? execution.config.defaultAnswerer)
+            ? resolveAnswererReference(
+                { prompt: "" },
+                profile.answerer ?? execution.config.defaultAnswerer,
+                harness,
+              )
             : undefined;
-        const answererCli = referencedAnswererCli(answerer);
-        const driverUserBehaviorHash =
-          driverCli?.name === "claude-code" ? await behaviorHash(claudeConfigRoot(driver)) : null;
-        const driverConfiguredSources = await configuredClaudeSourceHashes(
-          driver,
-          execution.config.rootDir,
-          configuredSourceHashes,
-        );
+        const answererCli = driver || harness ? referencedAnswererCli(answerer) : undefined;
         let harnessCommand = harness?.command;
         if (!harnessCommand) {
           harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
         }
-        const capturesHarnessIdentity =
-          Boolean(harness) || execution.loaded.campaign.evaluation.kind === "grader";
+        const capturesHarnessIdentity = Boolean(harness);
         const harnessCliIdentity = capturesHarnessIdentity
           ? await cliIdentity(harnessCommand, execution.config.rootDir)
           : null;
@@ -80,17 +70,15 @@ export const resolvedProfileSnapshot = async (
           profileId,
           {
             id: profileId,
+            environment: profile.environment ?? execution.config.environment ?? null,
+            configuredConcurrency: execution.config.concurrency,
+            evaluationConcurrency: 1,
             driver: driver ?? null,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            ...(driverCli?.name === "claude-code"
-              ? {
-                  driverClaudeConfigRoot: claudeConfigRoot(driver),
-                  driverUserBehaviorHash,
-                  driverConfiguredSources,
-                }
-              : {}),
+            driverIdentity: await experimentDriverIdentity(
+              driver,
+              execution.registry,
+              execution.config.rootDir,
+            ),
             answerer,
             answererCliIdentity: answererCli
               ? await cliIdentity(answererCli.command, execution.config.rootDir)
@@ -130,7 +118,11 @@ const trustedExecutableEntrypoints = async (
   }
   for (const profileId of execution.loaded.campaign.profiles) {
     const profile = profileOrThrow(execution.config, profileId);
-    add(referencedCli(profile.driver ?? execution.config.defaultDriver)?.command);
+    for (const command of referencedDriverRuntime(
+      profile.driver ?? execution.config.defaultDriver,
+      execution.registry,
+    ).commands)
+      add(command);
     add(referencedAnswererCli(profile.answerer ?? execution.config.defaultAnswerer)?.command);
     add((profile.harness ?? execution.config.harness)?.command);
   }
@@ -147,7 +139,20 @@ const trustedExecutableEntrypoints = async (
         await loadEvalCase(evalCase.path),
         execution.loaded.subjectRoot,
       );
-      add(referencedCli(authored.driver)?.command);
+      for (const profileId of execution.loaded.campaign.profiles) {
+        const profile = profileOrThrow(execution.config, profileId);
+        const driver = bindSubjectRoot(
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
+          execution.loaded.subjectRoot,
+        );
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command);
+      }
       add(referencedAnswererCli(authored.answerer)?.command);
       for (const reference of caseRubricCliReferences(execution, authored)) {
         add(reference.command);
@@ -182,13 +187,28 @@ export const runtimeCliReferences = async (
     );
   }
   if (corpus.kind === "grader") {
-    for (const profileId of execution.loaded.campaign.profiles) {
-      const profile = profileOrThrow(execution.config, profileId);
-      const harness = profile.harness ?? execution.config.harness;
-      add(
-        harness?.command ?? (harness?.provider === "codex" ? "codex" : "claude"),
-        execution.config.rootDir,
-      );
+    const annotations = execution.loaded.graderAnnotations?.annotations ?? [];
+    const selected = new Map(
+      selectorsBySplit(execution.loaded.campaign)
+        .flatMap((split) => selectGraderAnnotations(annotations, split.selector))
+        .map((annotation) => [annotation.id, annotation]),
+    );
+    for (const annotation of selected.values()) {
+      const run = await loadFrozenRegradableRun(annotation.runDir);
+      try {
+        const rubric = await loadRubricDefinition(
+          resolve(execution.loaded.subjectRoot, annotation.rubricPath),
+        );
+        const authored = {
+          ...run.case,
+          assertions: rubric.assertions,
+          driver: run.metadata.driver.name,
+        };
+        for (const reference of caseRubricCliReferences(execution, authored))
+          add(reference.command, execution.config.rootDir);
+      } finally {
+        await releaseFrozenRegradableRun(run);
+      }
     }
   }
   if (corpus.kind === "agent") {
@@ -206,17 +226,29 @@ export const runtimeCliReferences = async (
       );
       for (const profileId of execution.loaded.campaign.profiles) {
         const profile = profileOrThrow(execution.config, profileId);
-        const harness = profile.harness ?? execution.config.harness;
-        const driver = resolveDriverReference(
-          authored,
-          profile.driver ?? execution.config.defaultDriver,
+        const driver = bindSubjectRoot(
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
+          execution.loaded.subjectRoot,
         );
-        const answerer = resolveAnswererReference(
-          authored,
-          profile.answerer ?? execution.config.defaultAnswerer,
-          harness,
+        const harness = resolveHarnessForDriver(
+          profile.harness ?? execution.config.harness,
+          driver,
+        );
+        const answerer = bindSubjectRoot(
+          resolveAnswererReference(
+            authored,
+            profile.answerer ?? execution.config.defaultAnswerer,
+            harness,
+          ),
+          execution.loaded.subjectRoot,
         );
-        add(referencedCli(driver)?.command, execution.config.rootDir);
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command, execution.config.rootDir);
         add(referencedAnswererCli(answerer)?.command, execution.config.rootDir);
       }
       for (const reference of caseRubricCliReferences(execution, authored)) {
@@ -262,9 +294,9 @@ const authoredConfigDependencyHash = async (
   }
   return authoredDependencyGraphHash(configRoot, [...entrypoints]);
 };
-const validateAgentCaseProfileDrivers = async (
+export const validateAgentCaseProfileDrivers = async (
   execution: CampaignExecutionOptions,
-  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+  corpus: Pick<Extract<EvaluationCorpus, { kind: "agent" }>, "cases">,
 ): Promise<void> => {
   const selected = [
     ...new Map(
@@ -280,13 +312,17 @@ const validateAgentCaseProfileDrivers = async (
     );
     for (const profileId of execution.loaded.campaign.profiles) {
       const profile = profileOrThrow(execution.config, profileId);
-      const reference = resolveDriverReference(
-        authored,
-        profile.driver ?? execution.config.defaultDriver,
+      const reference = bindSubjectRoot(
+        resolveDriverReference(
+          authored,
+          profile.driver ?? execution.config.defaultDriver,
+          undefined,
+          execution.config.driverDefaults,
+        ),
+        execution.loaded.subjectRoot,
       );
       const name = typeof reference === "string" ? reference : reference.name;
       const rawConfig = typeof reference === "object" ? reference.config : undefined;
-      boundExperimentDriverConfig(name, rawConfig, "identity-check");
       const plugin = execution.registry.getDriver(name);
       if (!plugin) {
         throw new Error(
@@ -298,7 +334,7 @@ const validateAgentCaseProfileDrivers = async (
         rawConfig,
         `case '${evalCase.id}' profile '${profileId}' driver '${name}'`,
       );
-      boundExperimentDriverConfig(name, validated, "identity-check");
+      boundExperimentDriverConfig(plugin, validated, "identity-check");
     }
   }
 };
@@ -334,6 +370,7 @@ export const evaluationIdentity = async (
     luxVersion: LUX_VERSION,
     evaluatorImplementationHash: await evaluatorImplementationHash(),
     runtime: runtimeIdentity(),
+    concurrency: execution.config.concurrency,
     corpusHash: corpus.hash,
     profiles,
     configSourceHash,

package/src/orchestrator/experiment-preflight.ts

@@ -0,0 +1,27 @@
+import { assertValidationArgumentsBound } from "./candidates.js";
+import {
+  type CampaignExecutionOptions,
+  type CorpusSelection,
+  profileOrThrow,
+} from "./experiment-context.js";
+import { inspectCorpus, prepareCorpus } from "./experiment-corpus.js";
+import { evaluationIdentity, validateAgentCaseProfileDrivers } from "./experiment-identity.js";
+
+export const preparePlanningCorpus = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+  staticChecks: boolean,
+): Promise<CorpusSelection> => {
+  if (!staticChecks) {
+    const corpus = await prepareCorpus(execution);
+    await evaluationIdentity(execution, corpus, mode);
+    return corpus;
+  }
+  const corpus = await inspectCorpus(execution);
+  const campaign = execution.loaded.campaign;
+  for (const profileId of campaign.profiles) profileOrThrow(execution.config, profileId);
+  if (corpus.kind === "agent") await validateAgentCaseProfileDrivers(execution, corpus);
+  for (const validation of campaign.subject.validate)
+    assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+  return corpus;
+};

package/src/orchestrator/experiment-report.ts

@@ -2,6 +2,7 @@ import { basename } from "node:path";
 import { z } from "zod";
 import { valueHash } from "../core/index.js";
 import { deriveExperimentSelection, rankCandidates } from "./experiment.js";
+import { ExperimentEffectsSchema, experimentEffects } from "./experiment-effects.js";
 import {
   type ExperimentRun,
   ExperimentRunSchema,
@@ -24,6 +25,7 @@ export const ExperimentDecisionReportSchema = z
     status: z.enum(["running", "passed", "failed", "aborted"]),
     outcome: z.enum(["promote", "retain", "reject", "error", "incomplete"]),
     error: z.string().nullable(),
+    effects: z.exactOptional(ExperimentEffectsSchema),
     candidateAndAccountingVerified: z.boolean(),
     championCandidateId: z.string().nullable(),
     ranking: z.array(z.string()),
@@ -39,6 +41,7 @@ export const ExperimentDecisionReportSchema = z
       z
         .object({
           metric: z.string(),
+          relativeTo: z.exactOptional(z.literal("baseline")),
           profile: z.string().nullable(),
           split: ExperimentReportSplitSchema,
           passed: z.boolean(),
@@ -596,6 +599,7 @@ const projectExperimentDecisionReport = (
     evaluationKind: experiment.evaluationKind,
     status: experiment.status,
     error: experiment.errorMessage ?? null,
+    effects: experimentEffects(experiment),
     candidateAndAccountingVerified,
     outcome: reportOutcome(experiment, championId),
     championCandidateId: championId,
@@ -610,6 +614,7 @@ const projectExperimentDecisionReport = (
       .filter((constraint) => constraint.candidateId === targetId)
       .map((constraint) => ({
         metric: constraint.metric,
+        ...(constraint.relativeTo ? { relativeTo: constraint.relativeTo } : {}),
         profile: constraint.profile ?? null,
         split: constraint.split,
         passed: constraint.passed,
@@ -733,7 +738,7 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       "| --- | --- | --- | --- | ---: | --- | --- |",
       ...report.gates.map(
         (gate) =>
-          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
+          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)}${gate.relativeTo ? " Δ baseline" : ""} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
       ),
     );
   }
@@ -763,6 +768,21 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       ),
     );
   }
+  if (report.effects?.length) {
+    lines.push(
+      "",
+      "## Paired case effects",
+      "",
+      "Differences are candidate minus baseline; repetitions are averaged within cases and authored clusters receive equal weight.",
+      "",
+      "| Candidate | Profile | Split | Metric | Mean difference | Cases | Clusters | Confidence | Note |",
+      "| --- | --- | --- | --- | ---: | ---: | ---: | --- | --- |",
+      ...report.effects.map(
+        (effect) =>
+          `| ${markdown(effect.candidateId)} | ${markdown(effect.profileId)} | ${effect.split} | ${markdown(effect.metric)} | ${effect.meanDelta === null ? "—" : signed(effect.meanDelta)} | ${effect.caseCount ?? effect.caseIds.length} | ${effect.n} | ${markdown(intervalText(effect.interval ?? undefined) ?? "—")} | ${markdown(effect.note ?? "—")} |`,
+      ),
+    );
+  }
   lines.push(
     "",
     "## Source and spend",

package/src/orchestrator/experiment-runs.ts

@@ -86,6 +86,7 @@ export const ExperimentSelectionSchema = z.object({
     z.object({
       candidateId: CandidateIdSchema,
       metric: z.string(),
+      relativeTo: z.exactOptional(z.literal("baseline")),
       profile: z.exactOptional(z.string()),
       split: z.enum(["train", "validation", "test"]),
       values: z.array(z.number()),
@@ -738,6 +739,7 @@ export const resumeExperimentRun = async (path: string): Promise<OpenedExperimen
 
 export const listExperimentRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<ExperimentRun & { experimentDir: string }>> => {
   const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
   if (!existsSync(root)) return [];
@@ -745,10 +747,15 @@ export const listExperimentRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const runs = await Promise.all(
-    directories.map(async (experimentDir) => ({
-      experimentDir,
-      experiment: await loadExperimentRun(experimentDir),
-    })),
+    directories.map(async (experimentDir) => {
+      try {
+        return { experimentDir, experiment: await loadExperimentRun(experimentDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(experimentDir, error);
+        return { experimentDir, experiment: null };
+      }
+    }),
   );
   return runs
     .flatMap(({ experimentDir, experiment }) =>

package/src/orchestrator/experiment-runtime-identity.ts

@@ -1,61 +1,14 @@
 import { createHash } from "node:crypto";
-import { access, constants, lstat, open, readFile, realpath } from "node:fs/promises";
-import { homedir } from "node:os";
+import { access, constants, lstat, open, realpath } from "node:fs/promises";
 import { delimiter, isAbsolute, join, resolve } from "node:path";
 import { z } from "zod";
-import { contentHash, discoverCliVersion, valueHash } from "../core/index.js";
-import { walkAuthoredFiles } from "./fs-read.js";
-import { boundExperimentDriverConfig } from "./orchestrator.js";
+import { discoverCliVersion, valueHash } from "../core/index.js";
 
 export { LUX_VERSION } from "./package-version.js";
 
-const ClaudeSettingsFileSchema = z.looseObject({
-  enabledPlugins: z.record(z.string(), z.boolean()).optional(),
-});
-const ClaudeInstalledPluginRecordSchema = z.looseObject({
-  installPath: z.string(),
-});
-const ClaudeInstalledPluginsFileSchema = z.looseObject({
-  plugins: z.record(z.string(), z.array(ClaudeInstalledPluginRecordSchema)),
-});
-
-export const referencedCli = (
-  reference: unknown,
-): { name: string; command: string } | undefined => {
-  let name: string | undefined;
-  if (typeof reference === "string") {
-    name = reference;
-  } else if (
-    typeof reference === "object" &&
-    reference !== null &&
-    typeof (reference as { name?: unknown }).name === "string"
-  ) {
-    name = (reference as { name: string }).name;
-  }
-  const config =
-    typeof reference === "object" && reference !== null
-      ? (reference as { config?: unknown }).config
-      : undefined;
-  if (name === "subprocess") {
-    if (
-      typeof config !== "object" ||
-      config === null ||
-      typeof (config as { command?: unknown }).command !== "string"
-    ) {
-      return undefined;
-    }
-    return { name, command: (config as { command: string }).command };
-  }
-  if (name !== "claude-code" && name !== "codex") return undefined;
-  let command = name === "codex" ? "codex" : "claude";
-  if (
-    typeof config === "object" &&
-    config !== null &&
-    typeof (config as { command?: unknown }).command === "string"
-  ) {
-    command = (config as { command: string }).command;
-  }
-  return { name, command };
+const driverConfigRecord = (reference: unknown): Record<string, unknown> => {
+  const config = (reference as { config?: unknown } | null)?.config;
+  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
 };
 
 export const referencedAnswererCli = (
@@ -261,197 +214,6 @@ export const cliIdentity = async (command: string, baseRoot: string): Promise<Cl
   });
 };
 
-const optionalRegularFileHash = async (path: string): Promise<string | null> => {
-  try {
-    const info = await lstat(path);
-    if (info.isSymbolicLink() || !info.isFile()) {
-      throw new Error(`behavior-affecting settings must be a regular file: ${path}`);
-    }
-    return contentHash(await readFile(path));
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
-    throw error;
-  }
-};
-
-const driverConfigRecord = (driver: unknown): Record<string, unknown> => {
-  if (!driver || typeof driver !== "object") return {};
-  const config = (driver as { config?: unknown }).config;
-  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
-};
-
-export const claudeConfigRoot = (driver: unknown): string => {
-  const config = driverConfigRecord(driver);
-  const env =
-    config.env && typeof config.env === "object"
-      ? (config.env as Record<string, unknown>)
-      : undefined;
-  const configured = env?.CLAUDE_CONFIG_DIR;
-  const root =
-    typeof configured === "string"
-      ? configured
-      : (process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"));
-  if (!isAbsolute(root)) {
-    throw new Error("CLAUDE_CONFIG_DIR must be absolute for reproducible experiment identity");
-  }
-  return resolve(root);
-};
-
-const hashBehaviorDirectory = async (
-  root: string,
-  label: string,
-): Promise<Array<{ path: string; hash: string }>> => {
-  try {
-    const info = await lstat(root);
-    if (info.isSymbolicLink() || !info.isDirectory()) {
-      throw new Error(`behavior-affecting Claude source must be a real directory: ${root}`);
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
-    throw error;
-  }
-  const paths = await walkAuthoredFiles(root);
-  if (paths.length > 5_000) {
-    throw new Error(`behavior-affecting Claude source '${label}' exceeds 5000 files`);
-  }
-  let bytes = 0;
-  const hashes: Array<{ path: string; hash: string }> = [];
-  for (const path of paths) {
-    const content = await readFile(join(root, path));
-    bytes += content.length;
-    if (bytes > 256 * 1024 * 1024) {
-      throw new Error(`behavior-affecting Claude source '${label}' exceeds 256 MiB`);
-    }
-    hashes.push({ path: `${label}/${path}`, hash: contentHash(content) });
-  }
-  return hashes;
-};
-
-/**
- * Hash every intentionally loaded user behavior source without persisting its
- * contents. This covers user instructions/rules/output styles and the exact
- * installed contents of plugins enabled by settings.json.
- */
-export const claudeUserBehaviorHash = async (configRoot: string): Promise<string | null> => {
-  const settingsPath = join(configRoot, "settings.json");
-  const settingsHash = await optionalRegularFileHash(settingsPath);
-  const sources: Array<{ path: string; hash: string }> = [];
-  if (settingsHash) sources.push({ path: "settings.json", hash: settingsHash });
-  const claudeMdHash = await optionalRegularFileHash(join(configRoot, "CLAUDE.md"));
-  if (claudeMdHash) sources.push({ path: "CLAUDE.md", hash: claudeMdHash });
-  for (const directory of ["rules", "output-styles", "agents", "commands", "skills"]) {
-    sources.push(...(await hashBehaviorDirectory(join(configRoot, directory), directory)));
-  }
-
-  let settings: unknown;
-  try {
-    const contents = await readFile(settingsPath, "utf8");
-    try {
-      settings = JSON.parse(contents);
-    } catch {
-      throw new Error("Claude settings.json contains invalid JSON");
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      settings = undefined;
-    } else {
-      throw error;
-    }
-  }
-  const parsedSettings = ClaudeSettingsFileSchema.safeParse(settings);
-  if (settings !== undefined && !parsedSettings.success) {
-    throw new Error("Claude settings.json has an invalid enabledPlugins map");
-  }
-  const enabledPlugins = parsedSettings.success ? parsedSettings.data.enabledPlugins : undefined;
-  const enabledIds = Object.entries(enabledPlugins ?? {})
-    .filter(([, value]) => value === true)
-    .map(([id]) => id)
-    .sort();
-  if (enabledIds.length > 0) {
-    const installedPath = join(configRoot, "plugins", "installed_plugins.json");
-    const installedHash = await optionalRegularFileHash(installedPath);
-    if (installedHash) {
-      sources.push({ path: "plugins/installed_plugins.json", hash: installedHash });
-    }
-    let installed: unknown;
-    try {
-      installed = JSON.parse(await readFile(installedPath, "utf8"));
-    } catch {
-      installed = undefined;
-    }
-    const parsedInstalled = ClaudeInstalledPluginsFileSchema.safeParse(installed);
-    if (!parsedInstalled.success) {
-      throw new Error("enabled Claude plugins have invalid installed metadata");
-    }
-    const plugins = parsedInstalled.data.plugins;
-    for (const id of enabledIds) {
-      const records = plugins[id];
-      if (!records || records.length === 0) {
-        throw new Error(`enabled Claude plugin '${id}' has no installed content to hash`);
-      }
-      for (const [index, record] of records.entries()) {
-        const installPath = record.installPath;
-        sources.push(
-          ...(await hashBehaviorDirectory(resolve(installPath), `plugins/enabled/${id}/${index}`)),
-        );
-      }
-    }
-  }
-  return sources.length === 0 ? null : valueHash(sources);
-};
-
-export const configuredClaudeSourceHashes = async (
-  driver: unknown,
-  baseRoot: string,
-  cache: Map<string, Promise<string>> = new Map(),
-): Promise<Array<{ kind: "plugin" | "read"; path: string; hash: string }>> => {
-  const driverCli = referencedCli(driver);
-  if (!driverCli) return [];
-  const config = driverConfigRecord(driver);
-  // This also rejects path-bearing extraArgs that would evade the source
-  // hashes below. Runtime applies the same guard before driver creation.
-  boundExperimentDriverConfig(driverCli.name, config, "identity-check");
-  const isolation =
-    config.isolation && typeof config.isolation === "object"
-      ? (config.isolation as Record<string, unknown>)
-      : undefined;
-  const configured = [
-    ...(driverCli.name === "claude-code" && Array.isArray(config.pluginDirs)
-      ? config.pluginDirs.map((path) => ({ kind: "plugin" as const, path }))
-      : []),
-    ...(Array.isArray(config.addDirs)
-      ? config.addDirs.map((path) => ({ kind: "read" as const, path }))
-      : []),
-    ...(driverCli.name === "claude-code" && isolation && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead.map((path) => ({ kind: "read" as const, path }))
-      : []),
-  ].filter(
-    (entry): entry is { kind: "plugin" | "read"; path: string } =>
-      typeof entry.path === "string" && entry.path.length > 0,
-  );
-  return Promise.all(
-    configured
-      .sort((a, b) => `${a.kind}:${a.path}`.localeCompare(`${b.kind}:${b.path}`))
-      .map(async ({ kind, path }) => {
-        const absolute = resolve(baseRoot, path);
-        const key = `${kind}:${absolute}`;
-        let hash = cache.get(key);
-        if (!hash) {
-          hash = hashBehaviorDirectory(
-            absolute,
-            `driver/${kind}/${contentHash(absolute).slice(0, 12)}`,
-          ).then(valueHash);
-          cache.set(key, hash);
-        }
-        return {
-          kind,
-          path: absolute,
-          hash: await hash,
-        };
-      }),
-  );
-};
-
 export const runtimeCliIdentityHashForReferences = async (
   references: readonly RuntimeCliReference[],
 ): Promise<string> =>