@prismatic-io/lux 0.0.2-preview.14 → 0.0.2-preview.16

package/src/cli/render/reporter.ts

@@ -19,6 +19,7 @@ import {
   TurnUsageSchema,
   totalTokens,
 } from "../../core/index.js";
+import { hasAgentUsage } from "../../core/usage.js";
 import type { CaseObserver, SuiteCaseResult, SuiteObserver } from "../../orchestrator/index.js";
 import {
   formatAlignedUsage,
@@ -80,6 +81,7 @@ type CaseView = {
   assertions: GradingAssertionEntry[];
   skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>;
   usage: RunUsage;
+  agentUsageReported: boolean;
   /** Per-API-call agent usage, in order — the split behind `usage.agent`. */
   agentTurns: TurnUsage[];
   phaseDurationMs: PhaseDurations | null;
@@ -159,6 +161,7 @@ const cell = (text: string, width: number): string => {
  */
 const scoreNote = (entry: GradingAssertionEntry): string => {
   if (entry.errorMessage) return entry.errorMessage;
+  if (entry.abstention) return "insufficient evidence";
   if (entry.passed) return "";
   const score = formatScore(entry.score);
   if (entry.threshold > 0) return `score ${score} < ${formatScore(entry.threshold)}`;
@@ -240,6 +243,7 @@ export class ConsoleReporter implements SuiteObserver {
       assertions: [],
       skippedAssertions: [],
       usage: emptyRunUsage(),
+      agentUsageReported: false,
       agentTurns: [],
       phaseDurationMs: null,
       runDir: null,
@@ -273,6 +277,13 @@ export class ConsoleReporter implements SuiteObserver {
       view.toolUnresolved = tools.unresolved;
       view.toolUnmatchedResults = tools.unmatchedResults;
       view.toolMalformed = tools.malformedEvents;
+    } else if (event.type === "progress" && event.kind === "driver-stderr" && this.verbose) {
+      const payload = event.payload as { text?: unknown; truncated?: unknown } | null;
+      if (payload && typeof payload.text === "string") {
+        this.region.staticLine(this.theme.dim(` [${view.caseId}] ${payload.text}`));
+        if (payload.truncated === true)
+          this.region.staticLine(this.theme.dim(" [driver stderr truncated]"));
+      }
     } else if (event.type === "progress" && event.kind === "usage") {
       const turn = TurnUsageSchema.safeParse(event.payload).data;
       if (turn) view.agentTurns.push(turn);
@@ -307,6 +318,7 @@ export class ConsoleReporter implements SuiteObserver {
       view.assertions = result.result.grading.assertions;
       view.skippedAssertions = result.result.grading.skippedAssertions ?? [];
       view.usage = result.result.metadata.usage;
+      view.agentUsageReported = hasAgentUsage(result.result.metadata);
       view.phaseDurationMs = result.result.metadata.phaseDurationMs ?? null;
       const tools = result.result.metadata.toolOutcomes;
       if (tools) {
@@ -451,10 +463,13 @@ export class ConsoleReporter implements SuiteObserver {
       Math.max(NAME_MIN, ...view.assertions.map((a) => a.name.length)),
     );
     for (const entry of view.assertions) {
-      const mark = entry.passed ? this.theme.green("✓") : this.theme.red("✗");
+      let mark = entry.passed ? this.theme.green("✓") : this.theme.red("✗");
+      if (entry.abstention) mark = this.theme.dim("?");
       const duration = formatDuration(entry.durationMs).padStart(DURATION_COL);
       const note = scoreNote(entry);
-      const suffix = note ? `  ${this.theme.red(note)}` : "";
+      const suffix = note
+        ? `  ${entry.abstention ? this.theme.dim(note) : this.theme.red(note)}`
+        : "";
       nodes.push({
         head: `${mark} ${cell(entry.name, width)}  ${this.theme.dim(duration)}${suffix}`,
         body: !entry.passed && entry.comment ? [this.theme.dim(entry.comment)] : [],
@@ -630,11 +645,17 @@ export class ConsoleReporter implements SuiteObserver {
       emptyRunUsage(),
     );
     const harness = harnessUsage(total);
-    if (isUsageEmpty(total.agent) && isUsageEmpty(harness)) {
+    const reported = this.cases.filter(
+      (view) => view.agentUsageReported || !isUsageEmpty(this.agentUsageOf(view)),
+    ).length;
+    if (reported === 0 && isUsageEmpty(total.agent) && isUsageEmpty(harness)) {
       return live ? [] : [` ${label("Tokens")}${this.theme.dim("none reported")}`];
     }
 
-    const agentText = isUsageEmpty(total.agent) ? this.theme.dim("none") : formatUsage(total.agent);
+    const missing = this.cases.length - reported;
+    const partial = missing > 0 ? ` (${missing} run${missing === 1 ? "" : "s"} unavailable)` : "";
+    const agentText =
+      reported === 0 ? this.theme.dim("unavailable") : `${formatUsage(total.agent)}${partial}`;
     if (live) {
       const harnessText = isUsageEmpty(harness)
         ? ""

package/src/cli/run-options.ts

@@ -1,4 +1,4 @@
-import { InvalidArgumentError } from "commander";
+import { Errors } from "incur";
 
 export type RunOptions = {
   tag?: string[];
@@ -15,22 +15,14 @@ export type RunOptions = {
   verbose?: boolean;
 };
 
-export const collectTags = (value: string, previous: string[]): string[] =>
-  previous.concat(
-    value
-      .split(",")
-      .map((tag) => tag.trim())
-      .filter(Boolean),
-  );
-
-export const collectValues = (value: string, previous: string[]): string[] => [...previous, value];
-
 export const parseAnnotationAssertion = (
   value: string,
 ): [string, { passed: boolean; score?: number }] => {
   const match = /^(.+)=(pass|fail)(?::(.+))?$/.exec(value);
-  if (!match?.[1] || !match[2]) {
-    throw new InvalidArgumentError("--assertion must use id=pass, id=fail, or id=pass:0.8 syntax");
+  if (!match?.[1]?.trim() || !match[2]) {
+    throw new Errors.ParseError({
+      message: "--assertion must use id=pass, id=fail, or id=pass:0.8 syntax",
+    });
   }
   const score = match[3] === undefined ? undefined : parseUnitInterval(match[3]);
   return [
@@ -42,19 +34,10 @@ export const parseAnnotationAssertion = (
   ];
 };
 
-export const parsePositiveInt = (value: string): number => {
-  // Number() rejects partial numbers such as "3abc" and "2.9".
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new InvalidArgumentError("must be a positive integer");
-  }
-  return parsed;
-};
-
-export const parseUnitInterval = (value: string): number => {
+const parseUnitInterval = (value: string): number => {
   const parsed = Number(value);
-  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
-    throw new InvalidArgumentError("must be a number between 0 and 1");
+  if (!value.trim() || !Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new Errors.ParseError({ message: "must be a number between 0 and 1" });
   }
   return parsed;
 };

package/src/cli/view.ts

@@ -1,12 +1,6 @@
 import { resolve } from "node:path";
-import { InvalidArgumentError } from "commander";
+import { Errors } from "incur";
 import {
-  formatExperimentRun,
-  formatExperimentRunsTable,
-  formatRunDetail,
-  formatRunsTable,
-  formatSuiteRunDetail,
-  formatSuiteRunsTable,
   isExperimentRun,
   listExperimentRuns,
   listRuns,
@@ -29,7 +23,6 @@ export type ViewOptions = {
   web?: boolean;
   port?: number;
   open?: boolean;
-  json?: boolean;
 };
 
 const previewKind = async (
@@ -46,19 +39,16 @@ const previewKind = async (
   return undefined;
 };
 
-const webViewCommand = async (
+export async function* webViewCommand(
   runDirArg: string | undefined,
   options: ViewOptions,
-): Promise<void> => {
-  if (options.json) {
-    throw new InvalidArgumentError("--web and --json are mutually exclusive");
-  }
+): AsyncGenerator<{ url: string; runsRoot: string }, void> {
   const config = await loadLuxConfig(process.cwd());
   const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
   const initialPath = runDirArg ? resolve(runDirArg) : undefined;
-  if (initialPath && !(await ensureDirectory("view", initialPath))) return;
+  if (initialPath && !(await ensureDirectory("view", initialPath))) failWith(1);
   const initialKind = await previewKind(initialPath, options);
-  const controller = abortOnInterrupt();
+  using controller = abortOnInterrupt();
   const preview = await startPreviewServer({
     runsRoot,
     ...(options.port !== undefined ? { port: options.port } : {}),
@@ -67,30 +57,29 @@ const webViewCommand = async (
     ...(options.case ? { caseFilter: options.case } : {}),
     signal: controller.signal,
   });
-  process.stdout.write(`Lux Review: ${preview.url}\n`);
-  process.stdout.write(`Reading evidence from ${runsRoot}\n`);
-  if (options.open !== false) {
-    try {
-      await openPreviewUrl(preview.url);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      process.stderr.write(`lux view: could not open a browser (${message})\n`);
+  try {
+    yield { url: preview.url, runsRoot };
+    if (options.open !== false) {
+      try {
+        await openPreviewUrl(preview.url);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`lux view: could not open a browser (${message})\n`);
+      }
     }
+    await preview.closed;
+    if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+  } finally {
+    await preview.close();
   }
-  await preview.closed;
-  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
-};
+}
 
 export const viewCommand = async (
   runDirArg: string | undefined,
   options: ViewOptions,
-): Promise<void> => {
-  if (options.web) {
-    await webViewCommand(runDirArg, options);
-    return;
-  }
+): Promise<unknown> => {
   if (options.port !== undefined || options.open === false) {
-    throw new InvalidArgumentError("--port and --no-open require --web");
+    throw new Errors.ParseError({ message: "--port and --no-open require --web" });
   }
   if (runDirArg) {
     const path = resolve(runDirArg);
@@ -98,71 +87,43 @@ export const viewCommand = async (
     if (isExperimentRun(path)) {
       const experiment = await loadExperimentRun(path);
       if (!experiment) {
-        process.stderr.write(`lux view: invalid experiment at ${path}\n`);
-        failWith(1);
-        return;
+        throw new Errors.IncurError({
+          code: "INVALID_EXPERIMENT",
+          message: `Invalid experiment at ${path}.`,
+          exitCode: 1,
+        });
       }
-      process.stdout.write(
-        options.json
-          ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
-          : `${formatExperimentRun(path, experiment)}\n`,
-      );
-      return;
+      return { experimentDir: path, ...experiment };
     }
     const suite = await loadSuiteRun(path);
     if (suite) {
-      process.stdout.write(
-        options.json
-          ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
-          : `${formatSuiteRunDetail(path, suite)}\n`,
-      );
-      return;
+      return { suiteDir: path, ...suite };
     }
-    const detail = await loadRunDetail(path);
-    process.stdout.write(
-      options.json ? `${JSON.stringify(detail, null, 2)}\n` : `${formatRunDetail(detail)}\n`,
-    );
-    return;
+    return loadRunDetail(path);
   }
 
   const config = await loadLuxConfig(process.cwd());
   const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  const readErrors: Array<{ path: string; message: string }> = [];
+  const listingOptions = {
+    onReadError: (path: string, error: unknown) => {
+      readErrors.push({ path, message: error instanceof Error ? error.message : String(error) });
+    },
+  };
   if (options.experiments) {
-    const experiments = await listExperimentRuns(runsRoot);
-    process.stdout.write(
-      options.json
-        ? `${JSON.stringify(experiments, null, 2)}\n`
-        : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
-    );
-    return;
+    const experiments = await listExperimentRuns(runsRoot, listingOptions);
+    return { runsRoot, experiments, ...(readErrors.length ? { readErrors } : {}) };
   }
   if (options.suites) {
-    const suites = await listSuiteRuns(runsRoot);
-    process.stdout.write(
-      options.json
-        ? `${JSON.stringify(suites, null, 2)}\n`
-        : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
-    );
-    return;
+    const suites = await listSuiteRuns(runsRoot, listingOptions);
+    return { runsRoot, suites, ...(readErrors.length ? { readErrors } : {}) };
   }
   const all = await listRuns(runsRoot);
   const runs = options.case ? all.filter((run) => run.caseSlug === options.case) : all;
-  if (options.json) {
-    process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
-    return;
-  }
-  if (!options.case) {
-    process.stdout.write(`suite runs at ${runsRoot}:\n`);
-    process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
-  }
-  process.stdout.write(`runs at ${runsRoot}:\n`);
-  process.stdout.write(`${formatRunsTable(runs)}\n`);
-};
-
-export const parsePreviewPort = (value: string): number => {
-  const port = Number(value);
-  if (!Number.isInteger(port) || port < 0 || port > 65_535) {
-    throw new InvalidArgumentError("port must be an integer from 0 to 65535");
-  }
-  return port;
+  return {
+    runsRoot,
+    ...(!options.case ? { suites: await listSuiteRuns(runsRoot, listingOptions) } : {}),
+    runs,
+    ...(readErrors.length ? { readErrors } : {}),
+  };
 };

package/src/core/annotation.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
 import { formatZodIssues } from "./zod.js";
 
 const ExpectedBooleanLabelSchema = z
@@ -35,7 +36,7 @@ export const GraderAnnotationSchema = z
     reviewer: z.string().min(1).optional(),
     /** Human rationale supplied to the rubric optimizer on mismatches. */
     feedback: z.string().optional(),
-    meta: z.record(z.string(), z.unknown()).optional(),
+    meta: EvaluationMetadataSchema.optional(),
   })
   .strict()
   .superRefine((annotation, ctx) => {

package/src/core/answerer.ts

@@ -73,3 +73,12 @@ export type AnswererPlugin<Cfg = unknown> = {
   configSchema?: z.ZodTypeAny;
   create: (config: Cfg) => Answerer | Promise<Answerer>;
 };
+
+export const defineAnswerer = <S extends z.ZodType>(
+  definition: Omit<AnswererPlugin<z.output<S>>, "configSchema"> & { configSchema: S },
+): Omit<AnswererPlugin<z.output<S>>, "configSchema"> & { configSchema: S } => definition;
+
+export const answererRef = <S extends z.ZodType>(
+  plugin: { name: string; configSchema: S },
+  config: z.input<S>,
+): { name: string; config: z.input<S> } => ({ name: plugin.name, config });

package/src/core/assertion.ts

@@ -9,6 +9,8 @@ export const AssertionResultSchema = z.object({
   score: z.number().min(0).max(1),
   comment: z.exactOptional(z.string()),
   detail: z.unknown().optional(),
+  /** The evaluator ran successfully but lacked evidence for a verdict. */
+  abstention: z.exactOptional(z.literal("insufficient-evidence")),
   /** Present when the assertion could not produce a judgment. */
   errorMessage: z.exactOptional(z.string()),
   /** Tokens the assertion itself spent. Only LLM-judge assertions report this. */
@@ -143,6 +145,7 @@ export const GradingAssertionEntrySchema = z.object({
   durationMs: z.number(),
   comment: z.exactOptional(z.string()),
   detail: z.unknown().optional(),
+  abstention: z.exactOptional(z.literal("insufficient-evidence")),
   errorMessage: z.exactOptional(z.string()),
   usage: z.exactOptional(TokenUsageSchema),
 });

package/src/core/case.ts

@@ -1,6 +1,8 @@
 import { z } from "zod";
 import type { AssertionPlugin } from "./assertion.js";
 import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
+import { DriverCapabilitySchema } from "./driver.js";
+import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
 import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
 import { formatZodIssues } from "./zod.js";
 
@@ -46,8 +48,9 @@ const EvalCaseObjectSchema = z.object({
   driver: DriverRefSchema.optional(),
   answerer: AnswererRefSchema.optional(),
   fixtures: FixturesRefSchema.optional(),
+  requires: z.array(DriverCapabilitySchema).optional(),
   assertions: z.array(AssertionSchema).optional(),
-  meta: z.record(z.string(), z.unknown()).optional(),
+  meta: EvaluationMetadataSchema.optional(),
 });
 
 export const EvalCaseSchema = EvalCaseObjectSchema
@@ -149,6 +152,7 @@ export const PersistedCaseSchema = EvalCaseObjectSchema.partial()
     driver: PersistedDriverRefSchema.optional(),
     answerer: PersistedAnswererRefSchema.optional(),
     fixtures: PersistedFixturesRefSchema.optional(),
+    meta: z.record(z.string(), z.unknown()).optional(),
   })
   .loose();
 export type PersistedCase = z.infer<typeof PersistedCaseSchema>;

package/src/core/driver.ts

@@ -71,8 +71,46 @@ export interface AgentDriver {
   close(): Promise<void>;
 }
 
+export const DriverCapabilitySchema = z.enum([
+  "interactive",
+  "tool-calls",
+  "tool-results",
+  "artifacts",
+  "subagents",
+  "usage",
+  "cost",
+  "isolation",
+]);
+export type DriverCapability = z.infer<typeof DriverCapabilitySchema>;
+
+export type DriverRuntime = {
+  commands: string[];
+  /** Missing keys mean unknown; false means the adapter cannot provide the capability. */
+  capabilities: Partial<Record<DriverCapability, boolean>>;
+};
+
+export type DriverExperiment<Cfg> = {
+  version: string;
+  /** Bind a configuration to the experiment environment; Lux validates the returned config. */
+  prepare(config: Cfg): unknown;
+  /** Fingerprint behavior outside authored cases/configuration and the reported executables. */
+  identity(config: Cfg, context: { rootDir: string }): unknown | Promise<unknown>;
+};
+
 export type DriverPlugin<Cfg = unknown> = {
   name: string;
   configSchema?: z.ZodTypeAny;
   create: (config: Cfg) => AgentDriver | Promise<AgentDriver>;
+  runtime?: (config: Cfg) => DriverRuntime;
+  /** Ordinary runs remain supported when controlled experiments are not declared. */
+  experiment?: DriverExperiment<Cfg>;
 };
+
+export const defineDriver = <S extends z.ZodType>(
+  definition: Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S },
+): Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S } => definition;
+
+export const driverRef = <S extends z.ZodType>(
+  plugin: { name: string; configSchema: S },
+  config: z.input<S>,
+): { name: string; config: z.input<S> } => ({ name: plugin.name, config });

package/src/core/environment.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+/** Author-declared execution conditions. Lux records these without enforcing or verifying them. */
+export const EnvironmentDeclarationSchema = z
+  .object({
+    imageDigest: z
+      .string()
+      .regex(/^(?:[^@\s]+@)?sha256:[a-f0-9]{64}$/)
+      .optional(),
+    toolchains: z.record(z.string().min(1), z.string().min(1)).optional(),
+    network: z.enum(["disabled", "restricted", "unrestricted"]).optional(),
+    resources: z
+      .object({
+        cpuCount: z.number().positive().optional(),
+        memoryBytes: z.number().int().positive().optional(),
+        diskBytes: z.number().int().positive().optional(),
+      })
+      .strict()
+      .optional(),
+  })
+  .strict();
+export type EnvironmentDeclaration = z.infer<typeof EnvironmentDeclarationSchema>;

package/src/core/evaluation-clusters.ts

@@ -0,0 +1,35 @@
+import { z } from "zod";
+
+export const EvaluationMetadataSchema = z
+  .object({
+    /** Related perturbations share one independent statistical unit. */
+    cluster: z.string().trim().min(1).optional(),
+  })
+  .catchall(z.unknown());
+
+export const clusterKey = (id: string, cluster: string | undefined): string =>
+  JSON.stringify(cluster === undefined ? ["case", id] : ["cluster", cluster]);
+
+export const independentClusterCount = (
+  items: readonly { id: string; cluster?: string | undefined }[],
+): number => new Set(items.map((item) => clusterKey(item.id, item.cluster))).size;
+
+export const assertDisjointClusters = (
+  splits: readonly {
+    name: string;
+    items: readonly { id: string; cluster?: string | undefined }[];
+  }[],
+): void => {
+  const owners = new Map<string, string>();
+  for (const split of splits) {
+    for (const item of split.items) {
+      if (item.cluster === undefined) continue;
+      const owner = owners.get(item.cluster);
+      if (owner !== undefined && owner !== split.name)
+        throw new Error(
+          `evaluation cluster '${item.cluster}' appears in both ${owner} and ${split.name} splits`,
+        );
+      owners.set(item.cluster, split.name);
+    }
+  }
+};

package/src/core/experiment.ts

@@ -84,6 +84,8 @@ export type ExperimentVariant = z.output<typeof ExperimentVariantSchema>;
 export const ExperimentConstraintSchema = z
   .object({
     metric: z.string().min(1),
+    /** Bounds apply to paired candidate-minus-baseline case means. */
+    relativeTo: z.exactOptional(z.literal("baseline")),
     min: z.number().optional(),
     max: z.number().optional(),
     profile: z.string().optional(),
@@ -151,6 +153,47 @@ const DEFAULT_EXPERIMENT_BUDGET: ExperimentBudget = {
   maxWallTimeMs: 3_600_000,
 };
 
+const validateConstraintConfidence = (
+  constraint: ExperimentConstraint,
+  index: number,
+  boundedMetrics: Set<string>,
+  ctx: z.RefinementCtx,
+): void => {
+  const issue = (message: string): void =>
+    ctx.addIssue({ code: "custom", path: ["objectives", "constraints", index], message });
+  if (constraint.relativeTo && constraint.confidence?.method === "wilson") {
+    issue(
+      "paired differences are not Bernoulli observations; use Hoeffding or percentile-bootstrap confidence",
+    );
+  }
+  if (
+    constraint.relativeTo &&
+    ["subject.bytes", "quality.caseRegressions"].includes(constraint.metric)
+  ) {
+    issue("relative constraints require a metric with per-case observations");
+  }
+  const bounded = boundedMetrics.has(constraint.metric);
+  if (
+    (constraint.confidence?.method === "hoeffding" || constraint.confidence?.method === "wilson") &&
+    !bounded
+  ) {
+    issue(
+      `metric '${constraint.metric}' has no declared [0,1] support; use percentile-bootstrap confidence`,
+    );
+  }
+  const minimum = constraint.relativeTo ? -1 : 0;
+  if (
+    bounded &&
+    [constraint.min, constraint.max].some(
+      (bound) => bound !== undefined && (bound < minimum || bound > 1),
+    )
+  ) {
+    issue(
+      `metric '${constraint.metric}' has support [${minimum},1]; constraint bounds must stay in that interval`,
+    );
+  }
+};
+
 export const ExperimentCampaignSchema = z
   .object({
     id: z.string().min(1),
@@ -215,6 +258,8 @@ export const ExperimentCampaignSchema = z
       "grader.macroFalseNegativeRate",
       "grader.coverage",
       "grader.abstentionRate",
+      "grader.insufficientEvidenceRate",
+      "grader.operationalErrorRate",
       "grader.scoreMae",
     ]);
     const unitIntervalMetrics = graderConfidenceMetrics;
@@ -289,29 +334,7 @@ export const ExperimentCampaignSchema = z
       profiles.add(profile);
     }
     for (const [index, constraint] of campaign.objectives.constraints.entries()) {
-      if (
-        (constraint.confidence?.method === "hoeffding" ||
-          constraint.confidence?.method === "wilson") &&
-        !unitIntervalMetrics.has(constraint.metric)
-      ) {
-        ctx.addIssue({
-          code: "custom",
-          path: ["objectives", "constraints", index, "confidence", "method"],
-          message: `metric '${constraint.metric}' has no declared [0,1] support; use percentile-bootstrap confidence`,
-        });
-      }
-      if (
-        unitIntervalMetrics.has(constraint.metric) &&
-        [constraint.min, constraint.max].some(
-          (bound) => bound !== undefined && (bound < 0 || bound > 1),
-        )
-      ) {
-        ctx.addIssue({
-          code: "custom",
-          path: ["objectives", "constraints", index],
-          message: `metric '${constraint.metric}' has support [0,1]; constraint bounds must stay in that interval`,
-        });
-      }
+      validateConstraintConfidence(constraint, index, unitIntervalMetrics, ctx);
       if (constraint.profile && !profiles.has(constraint.profile)) {
         ctx.addIssue({
           code: "custom",
@@ -424,6 +447,20 @@ export const CandidateEvaluationSchema = z.object({
   costUsd: z.exactOptional(z.number().nonnegative()),
   /** Raw observations used by confidence-aware constraints. */
   samples: z.exactOptional(z.record(z.string(), z.array(z.number()))),
+  /** Independent cases with repetition coverage retained for paired comparisons. */
+  caseSamples: z.exactOptional(
+    z.record(
+      z.string(),
+      z.object({
+        identityHash: z.string().nullable(),
+        clusterId: z.exactOptional(z.string().min(1)),
+        metrics: z.record(
+          z.string(),
+          z.object({ mean: z.number().finite(), n: z.number().int().positive() }),
+        ),
+      }),
+    ),
+  ),
   failedCases: z.array(
     z.object({
       caseId: z.string(),

package/src/core/file-tree.ts

@@ -0,0 +1,35 @@
+import { existsSync } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+
+/**
+ * Walk authored trees without silently following or omitting a symlink that
+ * can change outside the experiment identity. Exact ignored paths support
+ * generated state directories whose names are project-configurable.
+ */
+export const walkAuthoredFiles = async (
+  root: string,
+  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
+  ignoredPaths: ReadonlySet<string> = new Set(),
+): Promise<string[]> => {
+  if (!existsSync(root)) return [];
+  const out: string[] = [];
+  const stack = [root];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    if (!dir) continue;
+    for (const entry of await readdir(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name);
+      if (ignoredPaths.has(resolve(full))) continue;
+      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
+      if (entry.isSymbolicLink()) {
+        throw new Error(
+          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
+        );
+      }
+      if (entry.isDirectory()) stack.push(full);
+      else if (entry.isFile()) out.push(relative(root, full));
+    }
+  }
+  return out.sort();
+};

package/src/core/index.ts

@@ -11,6 +11,7 @@ export * from "./conversation-turn.js";
 export * from "./driver.js";
 export * from "./driver-metrics.js";
 export * from "./emitter.js";
+export * from "./environment.js";
 export * from "./events.js";
 export * from "./exec.js";
 export * from "./experiment.js";