@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.32

package/src/orchestrator/experiment-runs.ts

@@ -86,6 +86,7 @@ export const ExperimentSelectionSchema = z.object({
     z.object({
       candidateId: CandidateIdSchema,
       metric: z.string(),
+      relativeTo: z.exactOptional(z.literal("baseline")),
       profile: z.exactOptional(z.string()),
       split: z.enum(["train", "validation", "test"]),
       values: z.array(z.number()),
@@ -738,6 +739,7 @@ export const resumeExperimentRun = async (path: string): Promise<OpenedExperimen
 
 export const listExperimentRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<ExperimentRun & { experimentDir: string }>> => {
   const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
   if (!existsSync(root)) return [];
@@ -745,10 +747,15 @@ export const listExperimentRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const runs = await Promise.all(
-    directories.map(async (experimentDir) => ({
-      experimentDir,
-      experiment: await loadExperimentRun(experimentDir),
-    })),
+    directories.map(async (experimentDir) => {
+      try {
+        return { experimentDir, experiment: await loadExperimentRun(experimentDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(experimentDir, error);
+        return { experimentDir, experiment: null };
+      }
+    }),
   );
   return runs
     .flatMap(({ experimentDir, experiment }) =>

package/src/orchestrator/experiment-runtime-identity.ts

@@ -1,61 +1,14 @@
 import { createHash } from "node:crypto";
-import { access, constants, lstat, open, readFile, realpath } from "node:fs/promises";
-import { homedir } from "node:os";
+import { access, constants, lstat, open, realpath } from "node:fs/promises";
 import { delimiter, isAbsolute, join, resolve } from "node:path";
 import { z } from "zod";
-import { contentHash, discoverCliVersion, valueHash } from "../core/index.js";
-import { walkAuthoredFiles } from "./fs-read.js";
-import { boundExperimentDriverConfig } from "./orchestrator.js";
+import { discoverCliVersion, valueHash } from "../core/index.js";
 
 export { LUX_VERSION } from "./package-version.js";
 
-const ClaudeSettingsFileSchema = z.looseObject({
-  enabledPlugins: z.record(z.string(), z.boolean()).optional(),
-});
-const ClaudeInstalledPluginRecordSchema = z.looseObject({
-  installPath: z.string(),
-});
-const ClaudeInstalledPluginsFileSchema = z.looseObject({
-  plugins: z.record(z.string(), z.array(ClaudeInstalledPluginRecordSchema)),
-});
-
-export const referencedCli = (
-  reference: unknown,
-): { name: string; command: string } | undefined => {
-  let name: string | undefined;
-  if (typeof reference === "string") {
-    name = reference;
-  } else if (
-    typeof reference === "object" &&
-    reference !== null &&
-    typeof (reference as { name?: unknown }).name === "string"
-  ) {
-    name = (reference as { name: string }).name;
-  }
-  const config =
-    typeof reference === "object" && reference !== null
-      ? (reference as { config?: unknown }).config
-      : undefined;
-  if (name === "subprocess") {
-    if (
-      typeof config !== "object" ||
-      config === null ||
-      typeof (config as { command?: unknown }).command !== "string"
-    ) {
-      return undefined;
-    }
-    return { name, command: (config as { command: string }).command };
-  }
-  if (name !== "claude-code" && name !== "codex") return undefined;
-  let command = name === "codex" ? "codex" : "claude";
-  if (
-    typeof config === "object" &&
-    config !== null &&
-    typeof (config as { command?: unknown }).command === "string"
-  ) {
-    command = (config as { command: string }).command;
-  }
-  return { name, command };
+const driverConfigRecord = (reference: unknown): Record<string, unknown> => {
+  const config = (reference as { config?: unknown } | null)?.config;
+  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
 };
 
 export const referencedAnswererCli = (
@@ -261,197 +214,6 @@ export const cliIdentity = async (command: string, baseRoot: string): Promise<Cl
   });
 };
 
-const optionalRegularFileHash = async (path: string): Promise<string | null> => {
-  try {
-    const info = await lstat(path);
-    if (info.isSymbolicLink() || !info.isFile()) {
-      throw new Error(`behavior-affecting settings must be a regular file: ${path}`);
-    }
-    return contentHash(await readFile(path));
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
-    throw error;
-  }
-};
-
-const driverConfigRecord = (driver: unknown): Record<string, unknown> => {
-  if (!driver || typeof driver !== "object") return {};
-  const config = (driver as { config?: unknown }).config;
-  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
-};
-
-export const claudeConfigRoot = (driver: unknown): string => {
-  const config = driverConfigRecord(driver);
-  const env =
-    config.env && typeof config.env === "object"
-      ? (config.env as Record<string, unknown>)
-      : undefined;
-  const configured = env?.CLAUDE_CONFIG_DIR;
-  const root =
-    typeof configured === "string"
-      ? configured
-      : (process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"));
-  if (!isAbsolute(root)) {
-    throw new Error("CLAUDE_CONFIG_DIR must be absolute for reproducible experiment identity");
-  }
-  return resolve(root);
-};
-
-const hashBehaviorDirectory = async (
-  root: string,
-  label: string,
-): Promise<Array<{ path: string; hash: string }>> => {
-  try {
-    const info = await lstat(root);
-    if (info.isSymbolicLink() || !info.isDirectory()) {
-      throw new Error(`behavior-affecting Claude source must be a real directory: ${root}`);
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
-    throw error;
-  }
-  const paths = await walkAuthoredFiles(root);
-  if (paths.length > 5_000) {
-    throw new Error(`behavior-affecting Claude source '${label}' exceeds 5000 files`);
-  }
-  let bytes = 0;
-  const hashes: Array<{ path: string; hash: string }> = [];
-  for (const path of paths) {
-    const content = await readFile(join(root, path));
-    bytes += content.length;
-    if (bytes > 256 * 1024 * 1024) {
-      throw new Error(`behavior-affecting Claude source '${label}' exceeds 256 MiB`);
-    }
-    hashes.push({ path: `${label}/${path}`, hash: contentHash(content) });
-  }
-  return hashes;
-};
-
-/**
- * Hash every intentionally loaded user behavior source without persisting its
- * contents. This covers user instructions/rules/output styles and the exact
- * installed contents of plugins enabled by settings.json.
- */
-export const claudeUserBehaviorHash = async (configRoot: string): Promise<string | null> => {
-  const settingsPath = join(configRoot, "settings.json");
-  const settingsHash = await optionalRegularFileHash(settingsPath);
-  const sources: Array<{ path: string; hash: string }> = [];
-  if (settingsHash) sources.push({ path: "settings.json", hash: settingsHash });
-  const claudeMdHash = await optionalRegularFileHash(join(configRoot, "CLAUDE.md"));
-  if (claudeMdHash) sources.push({ path: "CLAUDE.md", hash: claudeMdHash });
-  for (const directory of ["rules", "output-styles", "agents", "commands", "skills"]) {
-    sources.push(...(await hashBehaviorDirectory(join(configRoot, directory), directory)));
-  }
-
-  let settings: unknown;
-  try {
-    const contents = await readFile(settingsPath, "utf8");
-    try {
-      settings = JSON.parse(contents);
-    } catch {
-      throw new Error("Claude settings.json contains invalid JSON");
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      settings = undefined;
-    } else {
-      throw error;
-    }
-  }
-  const parsedSettings = ClaudeSettingsFileSchema.safeParse(settings);
-  if (settings !== undefined && !parsedSettings.success) {
-    throw new Error("Claude settings.json has an invalid enabledPlugins map");
-  }
-  const enabledPlugins = parsedSettings.success ? parsedSettings.data.enabledPlugins : undefined;
-  const enabledIds = Object.entries(enabledPlugins ?? {})
-    .filter(([, value]) => value === true)
-    .map(([id]) => id)
-    .sort();
-  if (enabledIds.length > 0) {
-    const installedPath = join(configRoot, "plugins", "installed_plugins.json");
-    const installedHash = await optionalRegularFileHash(installedPath);
-    if (installedHash) {
-      sources.push({ path: "plugins/installed_plugins.json", hash: installedHash });
-    }
-    let installed: unknown;
-    try {
-      installed = JSON.parse(await readFile(installedPath, "utf8"));
-    } catch {
-      installed = undefined;
-    }
-    const parsedInstalled = ClaudeInstalledPluginsFileSchema.safeParse(installed);
-    if (!parsedInstalled.success) {
-      throw new Error("enabled Claude plugins have invalid installed metadata");
-    }
-    const plugins = parsedInstalled.data.plugins;
-    for (const id of enabledIds) {
-      const records = plugins[id];
-      if (!records || records.length === 0) {
-        throw new Error(`enabled Claude plugin '${id}' has no installed content to hash`);
-      }
-      for (const [index, record] of records.entries()) {
-        const installPath = record.installPath;
-        sources.push(
-          ...(await hashBehaviorDirectory(resolve(installPath), `plugins/enabled/${id}/${index}`)),
-        );
-      }
-    }
-  }
-  return sources.length === 0 ? null : valueHash(sources);
-};
-
-export const configuredClaudeSourceHashes = async (
-  driver: unknown,
-  baseRoot: string,
-  cache: Map<string, Promise<string>> = new Map(),
-): Promise<Array<{ kind: "plugin" | "read"; path: string; hash: string }>> => {
-  const driverCli = referencedCli(driver);
-  if (!driverCli) return [];
-  const config = driverConfigRecord(driver);
-  // This also rejects path-bearing extraArgs that would evade the source
-  // hashes below. Runtime applies the same guard before driver creation.
-  boundExperimentDriverConfig(driverCli.name, config, "identity-check");
-  const isolation =
-    config.isolation && typeof config.isolation === "object"
-      ? (config.isolation as Record<string, unknown>)
-      : undefined;
-  const configured = [
-    ...(driverCli.name === "claude-code" && Array.isArray(config.pluginDirs)
-      ? config.pluginDirs.map((path) => ({ kind: "plugin" as const, path }))
-      : []),
-    ...(Array.isArray(config.addDirs)
-      ? config.addDirs.map((path) => ({ kind: "read" as const, path }))
-      : []),
-    ...(driverCli.name === "claude-code" && isolation && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead.map((path) => ({ kind: "read" as const, path }))
-      : []),
-  ].filter(
-    (entry): entry is { kind: "plugin" | "read"; path: string } =>
-      typeof entry.path === "string" && entry.path.length > 0,
-  );
-  return Promise.all(
-    configured
-      .sort((a, b) => `${a.kind}:${a.path}`.localeCompare(`${b.kind}:${b.path}`))
-      .map(async ({ kind, path }) => {
-        const absolute = resolve(baseRoot, path);
-        const key = `${kind}:${absolute}`;
-        let hash = cache.get(key);
-        if (!hash) {
-          hash = hashBehaviorDirectory(
-            absolute,
-            `driver/${kind}/${contentHash(absolute).slice(0, 12)}`,
-          ).then(valueHash);
-          cache.set(key, hash);
-        }
-        return {
-          kind,
-          path: absolute,
-          hash: await hash,
-        };
-      }),
-  );
-};
-
 export const runtimeCliIdentityHashForReferences = async (
   references: readonly RuntimeCliReference[],
 ): Promise<string> =>

package/src/orchestrator/experiment-selection.ts

@@ -13,6 +13,7 @@ import {
   ExperimentSelectionSchema,
   type OpenedExperimentRun,
 } from "./experiment-runs.js";
+import { confidenceGateIssue, type PairedEffect, pairedEffect } from "./paired-effects.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
 
@@ -103,6 +104,7 @@ const satisfiesConstraint = (
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): boolean => {
+  if (constraint.relativeTo && candidate.id === "baseline") return true;
   let profiles: (string | undefined)[];
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -110,6 +112,16 @@ const satisfiesConstraint = (
     profiles = campaign.profiles;
   }
   return profiles.every((profile) => {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return false;
+      const lower = effect.interval?.lower ?? effect.meanDelta;
+      const upper = effect.interval?.upper ?? effect.meanDelta;
+      return (
+        (constraint.min === undefined || lower >= constraint.min) &&
+        (constraint.max === undefined || upper <= constraint.max)
+      );
+    }
     const values = metricValues(
       candidate,
       constraint.metric,
@@ -135,6 +147,7 @@ const satisfiesConstraint = (
       return sampleGroups.every((samples) => {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return false;
           return (
             (constraint.min === undefined || interval.lower >= constraint.min) &&
             (constraint.max === undefined || interval.upper <= constraint.max)
@@ -156,12 +169,29 @@ const satisfiesConstraint = (
 export const average = (values: readonly number[]): number =>
   values.reduce((sum, value) => sum + value, 0) / Math.max(values.length, 1);
 
+const relativeConstraintEffect = (
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): PairedEffect => {
+  const scope = (entry: ExperimentEvaluation): boolean =>
+    entry.profileId === profile && entry.split === constraint.split;
+  return pairedEffect(
+    candidateEvaluations(experiment, "baseline").find(scope),
+    candidateEvaluations(experiment, candidate.id).find(scope),
+    constraint.metric,
+    constraint.confidence,
+  );
+};
+
 const normalizedConstraintViolation = (
   campaign: ExperimentCampaign,
   candidate: ExperimentDecisionCandidate,
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): number => {
+  if (constraint.relativeTo && candidate.id === "baseline") return 0;
   let profiles: Array<string | undefined> = campaign.profiles;
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -177,6 +207,18 @@ const normalizedConstraintViolation = (
     );
   let worst = 0;
   for (const profile of profiles) {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return Number.POSITIVE_INFINITY;
+      worst = Math.max(
+        worst,
+        violation(
+          effect.interval?.lower ?? effect.meanDelta,
+          effect.interval?.upper ?? effect.meanDelta,
+        ),
+      );
+      continue;
+    }
     if (constraint.confidence) {
       const groups = metricSampleGroups(
         candidate,
@@ -189,6 +231,7 @@ const normalizedConstraintViolation = (
       for (const samples of groups) {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return Number.POSITIVE_INFINITY;
           worst = Math.max(worst, violation(interval.lower, interval.upper));
         } catch {
           return Number.POSITIVE_INFINITY;
@@ -379,6 +422,36 @@ export const searchFrontier = (
   });
 };
 
+const relativeConstraintReport = (
+  campaign: ExperimentCampaign,
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): ExperimentSelection["constraints"][number] => {
+  const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+  const reference = candidate.id === "baseline";
+  let values = effect.meanDelta === null ? [] : [effect.meanDelta];
+  if (reference) values = [0];
+  const note = reference ? "baseline reference; relative gate does not apply" : effect.note;
+  return {
+    candidateId: candidate.id,
+    metric: constraint.metric,
+    relativeTo: "baseline",
+    ...(profile ? { profile } : {}),
+    split: constraint.split,
+    values,
+    passed: satisfiesConstraint(
+      campaign,
+      candidate,
+      { ...constraint, ...(profile ? { profile } : {}) },
+      experiment,
+    ),
+    ...(!reference && effect.interval ? { interval: effect.interval } : {}),
+    ...(note ? { note } : {}),
+  };
+};
+
 export const deriveExperimentSelection = (
   campaign: ExperimentCampaign,
   candidates: readonly ExperimentDecisionCandidate[],
@@ -405,6 +478,9 @@ export const deriveExperimentSelection = (
           ? [constraint.profile]
           : campaign.profiles;
       return profiles.map((profile) => {
+        if (constraint.relativeTo) {
+          return relativeConstraintReport(campaign, candidate, constraint, profile, experiment);
+        }
         const values = metricValues(
           candidate,
           constraint.metric,
@@ -426,6 +502,7 @@ export const deriveExperimentSelection = (
           if (samples) {
             try {
               interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+              note = confidenceGateIssue(interval);
             } catch (error) {
               note = error instanceof Error ? error.message : String(error);
             }

package/src/orchestrator/experiment.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import type { z } from "zod";
+import { independentClusterCount } from "../core/evaluation-clusters.js";
 import {
   bootstrapMeanConfidenceInterval,
   type Candidate,
@@ -47,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -55,14 +57,7 @@ import {
   openExperimentRun,
   resumeExperimentRun,
 } from "./experiment-runs.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  LUX_VERSION,
-  runtimeCliIdentityHashForReferences,
-} from "./experiment-runtime-identity.js";
+import { LUX_VERSION, runtimeCliIdentityHashForReferences } from "./experiment-runtime-identity.js";
 import {
   candidateEvaluations,
   type ExperimentSelectionDerivationOptionsSchema,
@@ -188,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -198,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({
@@ -254,50 +246,57 @@ export const planExperimentCampaign = async (
     const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
     let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
     if (corpus.kind === "grader" && splitSelector) {
-      const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+      let selected = selectGraderAnnotations(graderAnnotations, splitSelector);
       if (constraint.metric === "grader.macroFalsePositiveRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(false),
-        ).length;
+        );
       } else if (constraint.metric === "grader.macroFalseNegativeRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(true),
-        ).length;
+        );
       } else if (constraint.metric === "grader.scoreMae") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.caseScore !== undefined ||
             Object.values(annotation.expected.assertions).some(
               (label) => label.score !== undefined,
             ),
-        ).length;
+        );
       } else if (constraint.metric === "grader.caseAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             annotation.expected.caseScore !== undefined,
-        ).length;
+        );
       } else if (constraint.metric === "grader.assertionAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) => Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       } else if (
         constraint.metric === "grader.abstentionRate" ||
+        constraint.metric === "grader.insufficientEvidenceRate" ||
+        constraint.metric === "grader.operationalErrorRate" ||
         constraint.metric === "grader.coverage"
       ) {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       }
+      examples = independentClusterCount(
+        selected.map((annotation) => ({ id: annotation.id, cluster: annotation.meta?.cluster })),
+      );
     }
+    if (corpus.kind === "agent" && splitSelector)
+      examples = independentClusterCount(selectCases(corpus.cases, splitSelector));
     if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
       examples = 1;
     }
@@ -318,6 +317,10 @@ export const planExperimentCampaign = async (
         Array<number>(examples).fill(0),
         constraint.confidence,
       );
+      if (constraint.relativeTo) {
+        bestForMin.lower = 2 * bestForMin.lower - 1;
+        bestForMax.upper = 2 * bestForMax.upper - 1;
+      }
       if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
         notes.push(
           `${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`,
@@ -986,6 +989,3 @@ export const optimizeExperimentCampaign = async (
     new CampaignExecution(execution, "optimize", runOptimizationCampaignWork),
   );
 };
-
-// Internal exports for identity contract tests.
-export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };

package/src/orchestrator/fixture-identity.ts

@@ -5,6 +5,7 @@ import { contentHash, type FixturesRef, valueHash } from "../core/index.js";
 export type FixtureIdentity = {
   rootDir: string;
   hash: string;
+  fileHashes: ReadonlyMap<string, string>;
 };
 
 const fixtureEntries = async (
@@ -63,5 +64,12 @@ export const fixtureIdentity = async (
   if (metadata.isSymbolicLink() || !metadata.isDirectory()) {
     throw new Error(`fixtures: expected a real directory: ${rootDir}`);
   }
-  return { rootDir, hash: valueHash(await fixtureEntries(rootDir)) };
+  const entries = await fixtureEntries(rootDir);
+  return {
+    rootDir,
+    hash: valueHash(entries),
+    fileHashes: new Map(
+      entries.flatMap((entry) => (entry.hash === undefined ? [] : [[entry.path, entry.hash]])),
+    ),
+  };
 };

package/src/orchestrator/fs-read.ts

@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs";
 import type { FileHandle } from "node:fs/promises";
 import { readdir, readFile } from "node:fs/promises";
-import { join, relative, resolve } from "node:path";
+import { join, relative } from "node:path";
 import type { z } from "zod";
 
 const errorMessage = (error: unknown): string =>
@@ -93,34 +93,4 @@ export const walkFiles = async (root: string, maxFiles?: number): Promise<string
   return out.sort();
 };
 
-/**
- * Walk authored trees without silently following or omitting a symlink that
- * can change outside the experiment identity. Exact ignored paths support
- * generated state directories whose names are project-configurable.
- */
-export const walkAuthoredFiles = async (
-  root: string,
-  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
-  ignoredPaths: ReadonlySet<string> = new Set(),
-): Promise<string[]> => {
-  if (!existsSync(root)) return [];
-  const out: string[] = [];
-  const stack = [root];
-  while (stack.length > 0) {
-    const dir = stack.pop();
-    if (!dir) continue;
-    for (const entry of await readdir(dir, { withFileTypes: true })) {
-      const full = join(dir, entry.name);
-      if (ignoredPaths.has(resolve(full))) continue;
-      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
-      if (entry.isSymbolicLink()) {
-        throw new Error(
-          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
-        );
-      }
-      if (entry.isDirectory()) stack.push(full);
-      else if (entry.isFile()) out.push(relative(root, full));
-    }
-  }
-  return out.sort();
-};
+export { walkAuthoredFiles } from "../core/file-tree.js";