@prismatic-io/lux 0.0.2-preview.14 → 0.0.2-preview.16

package/src/orchestrator/compare.ts

@@ -17,6 +17,7 @@ import {
   stableJson,
   totalTokens,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { stableDriverIdentity } from "./comparison-identity.js";
 import { safeReadJson, walkFiles } from "./fs-read.js";
 
@@ -59,6 +60,7 @@ export type RunComparison = {
       b: { ask: number; approve: number; custom: number };
     };
     usage: { a: RunUsage; b: RunUsage };
+    agentUsageAvailable?: { a: boolean; b: boolean };
   } | null;
   grading: {
     casePassedA: boolean | null;
@@ -232,6 +234,11 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
   if (stableJson(runtimeA) !== stableJson(runtimeB)) {
     caseMismatch.push({ field: "runtime", a: runtimeA, b: runtimeB });
   }
+  const environmentA = a.runJson?.provenance?.environment;
+  const environmentB = b.runJson?.provenance?.environment;
+  if (stableJson(environmentA) !== stableJson(environmentB)) {
+    caseMismatch.push({ field: "environment", a: environmentA, b: environmentB });
+  }
   const sourceA = a.runJson?.provenance?.source;
   const sourceB = b.runJson?.provenance?.source;
   const sourceHashUnavailable =
@@ -282,6 +289,10 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
           },
+          agentUsageAvailable: {
+            a: a.runJson !== null && hasAgentUsage(a.runJson),
+            b: b.runJson !== null && hasAgentUsage(b.runJson),
+          },
           usage: {
             a: a.runJson?.usage ?? emptyRunUsage(),
             b: b.runJson?.usage ?? emptyRunUsage(),
@@ -350,7 +361,11 @@ export const formatComparisonText = (cmp: RunComparison): string => {
     );
     const au = cmp.runDelta.usage.a;
     const bu = cmp.runDelta.usage.b;
-    lines.push(`  agent tokens: ${totalTokens(au.agent)} → ${totalTokens(bu.agent)}`);
+    const agentA =
+      cmp.runDelta.agentUsageAvailable?.a === false ? "unavailable" : totalTokens(au.agent);
+    const agentB =
+      cmp.runDelta.agentUsageAvailable?.b === false ? "unavailable" : totalTokens(bu.agent);
+    lines.push(`  agent tokens: ${agentA} → ${agentB}`);
     lines.push(
       `  harness tokens: ${totalTokens(harnessUsage(au))} → ${totalTokens(harnessUsage(bu))}`,
     );

package/src/orchestrator/comparison-identity.ts

@@ -31,6 +31,7 @@ export const runComparisonIdentity = (metadata: RunMetadata): string =>
   valueHash({
     source: sourceIdentity(metadata),
     runtime: metadata.provenance?.runtime,
+    environment: metadata.provenance?.environment,
     caseHash: metadata.provenance?.caseHash,
     assertionsHash: metadata.provenance?.assertionsHash,
     fixturesHash: metadata.provenance?.fixturesHash,

package/src/orchestrator/config.ts

@@ -4,11 +4,11 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
 import { createRubricAssertion } from "../assertions/rubric/index.js";
+import { EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type AnswererPlugin,
   AnswererRefSchema,
@@ -25,6 +25,7 @@ import {
 } from "../core/index.js";
 import { claudeCodeDriver } from "../drivers/claude-code/index.js";
 import { codexDriver } from "../drivers/codex/index.js";
+import { cursorDriver } from "../drivers/cursor/index.js";
 import { mcpProbeDriver } from "../drivers/mcp/index.js";
 import { subprocessDriver } from "../drivers/subprocess/index.js";
 
@@ -35,6 +36,7 @@ const pluginLike = z.looseObject({
 
 export const ExperimentProfileSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     driver: DriverRefSchema.optional(),
     answerer: AnswererRefSchema.optional(),
     harness: HarnessModelConfigSchema.optional(),
@@ -70,6 +72,7 @@ const LifecycleHooksPluginSchema = z.custom<LifecycleHooksPlugin>().superRefine(
 
 const LuxConfigSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     casesRoot: z.string().optional(),
     fixturesRoot: z.string().optional(),
     runsRoot: z.string().default(".lux-runs"),
@@ -165,6 +168,7 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
     ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
     ...(validated.harness ? { harness: validated.harness } : {}),
     concurrency: validated.concurrency,
+    ...(validated.environment ? { environment: validated.environment } : {}),
     ...(validated.profiles ? { profiles: validated.profiles } : {}),
     ...(validated.drivers ? { drivers: validated.drivers } : {}),
     ...(validated.answerers ? { answerers: validated.answerers } : {}),
@@ -176,13 +180,13 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
 const builtinDrivers: DriverPlugin<never>[] = [
   claudeCodeDriver,
   codexDriver,
+  cursorDriver,
   subprocessDriver,
   mcpProbeDriver,
 ];
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {

package/src/orchestrator/discover.ts

@@ -13,6 +13,7 @@ export type DiscoveredCase = {
   relPath: string;
   /** `meta.tags` normalized to strings; empty when unset. */
   tags: string[];
+  cluster?: string;
 };
 
 /**
@@ -96,7 +97,13 @@ export const discoverCases = async (root: string): Promise<DiscoveryResult> => {
       problems.push({ path, reason: formatZodIssues(parsed.error) });
       continue;
     }
-    cases.push({ path, id: parsed.data.id ?? relPath, relPath, tags: tagsOf(parsed.data.meta) });
+    cases.push({
+      path,
+      id: parsed.data.id ?? relPath,
+      relPath,
+      tags: tagsOf(parsed.data.meta),
+      ...(parsed.data.meta?.cluster ? { cluster: parsed.data.meta.cluster } : {}),
+    });
   }
   return { cases, problems };
 };

package/src/orchestrator/doctor.ts

@@ -8,12 +8,14 @@ import type { EvalCase, Registry } from "../core/index.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
-import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
+import { validateDriverCapabilities } from "./driver-capabilities.js";
+import { ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
 import {
   experimentCampaignMode,
   loadExperimentCampaign,
   resolveExperimentCampaignPath,
 } from "./experiment-loader.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
 import { assertionConfigForSchema } from "./grading.js";
 import { loadEvalCase } from "./loader.js";
 import { validatePluginConfig, validatePluginConfigAsync } from "./validate-plugin-config.js";
@@ -142,17 +144,20 @@ const caseDriverReferences = (local: unknown, config: ResolvedLuxConfig): unknow
   ];
 };
 
-const driverCommand = (reference: unknown): string | undefined => {
+const driverCommands = (reference: unknown, registry: Registry): string[] => {
   const name = refName(reference);
-  const config = refConfig(reference);
-  if (name === "claude-code") {
-    return typeof config.command === "string" ? config.command : "claude";
-  }
-  if (name === "codex") return typeof config.command === "string" ? config.command : "codex";
-  if (name === "subprocess") {
-    return typeof config.command === "string" ? config.command : undefined;
+  const plugin = name ? registry.getDriver(name) : undefined;
+  if (!plugin?.runtime) return [];
+  try {
+    const config = validatePluginConfig(
+      plugin.configSchema,
+      rawRefConfig(reference),
+      `driver '${name}'`,
+    );
+    return plugin.runtime(config).commands;
+  } catch {
+    return [];
   }
-  return undefined;
 };
 
 const answererCommand = (reference: unknown): string | undefined => {
@@ -179,12 +184,13 @@ const rubricCommands = (assertion: unknown, config: ResolvedLuxConfig): string[]
   return [...new Set(commands)];
 };
 
-const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
+const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): CliTarget[] => {
   const targets: CliTarget[] = [];
   const add = (label: string, command: string | undefined): void => {
     if (command) targets.push({ label, command });
   };
-  add("default driver", driverCommand(config.defaultDriver));
+  for (const command of driverCommands(config.defaultDriver, registry))
+    add("default driver", command);
   add("default answerer", answererCommand(config.defaultAnswerer));
   if (config.harness) {
     add(
@@ -195,7 +201,9 @@ const configuredCliTargets = (config: ResolvedLuxConfig): CliTarget[] => {
   for (const [id, profile] of Object.entries(config.profiles ?? {}).sort(([a], [b]) =>
     a.localeCompare(b),
   )) {
-    add(`profile '${id}' driver`, driverCommand(profile.driver ?? config.defaultDriver));
+    for (const command of driverCommands(profile.driver ?? config.defaultDriver, registry)) {
+      add(`profile '${id}' driver`, command);
+    }
     add(`profile '${id}' answerer`, answererCommand(profile.answerer ?? config.defaultAnswerer));
     const harness = profile.harness ?? config.harness;
     if (harness) {
@@ -418,17 +426,19 @@ const caseDriverFailure = (
   caseId: string,
   reference: unknown,
   registry: Registry,
+  required: EvalCase["requires"],
 ): string | null => {
   const name = refName(reference);
   if (!name) return "driver reference has no name";
   const plugin = registry.getDriver(name);
   if (!plugin) return `unknown driver '${name}'`;
   try {
-    validatePluginConfig(
+    const config = validatePluginConfig(
       plugin.configSchema,
       rawRefConfig(reference),
       `case '${caseId}' driver '${name}'`,
     );
+    validateDriverCapabilities(plugin, config, required);
     return null;
   } catch (error) {
     return firstLine(error);
@@ -440,6 +450,7 @@ const validateCaseDrivers = (
   drivers: readonly unknown[],
   registry: Registry,
   recorder: DoctorRecorder,
+  required: EvalCase["requires"],
 ): void => {
   if (drivers.length === 0) {
     recorder.add(
@@ -450,7 +461,7 @@ const validateCaseDrivers = (
     return;
   }
   const failures = drivers
-    .map((driver) => caseDriverFailure(caseId, driver, registry))
+    .map((driver) => caseDriverFailure(caseId, driver, registry, required))
     .filter((failure): failure is string => failure !== null);
   if (failures.length === drivers.length) {
     recorder.add(
@@ -542,13 +553,16 @@ const executionTargetsForCase = (
   caseId: string,
   drivers: readonly unknown[],
   answerer: unknown,
+  registry: Registry,
 ): CliTarget[] => {
   const targets = drivers.flatMap((driver): CliTarget[] => {
-    const command = driverCommand(driver);
-    return command ? [{ label: `case '${caseId}' driver`, command }] : [];
+    return driverCommands(driver, registry).map((command) => ({
+      label: `case '${caseId}' driver`,
+      command,
+    }));
   });
-  const command = answererCommand(answerer);
-  if (command) targets.push({ label: `case '${caseId}' answerer`, command });
+  const answererCli = answererCommand(answerer);
+  if (answererCli) targets.push({ label: `case '${caseId}' answerer`, command: answererCli });
   return targets;
 };
 
@@ -563,11 +577,11 @@ const inspectCase = async (
     const caseRegistry = registryForEvalCase(registry, authored);
     const drivers = caseDriverReferences(authored.driver, config);
     const answerer = authored.answerer ?? config.defaultAnswerer;
-    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
+    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder, authored.requires);
     validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [
       ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
-      ...executionTargetsForCase(discovered.id, drivers, answerer),
+      ...executionTargetsForCase(discovered.id, drivers, answerer, caseRegistry),
     ];
   } catch (error) {
     recorder.add(
@@ -735,7 +749,11 @@ export const doctorProject = async (options: DoctorOptions): Promise<DoctorRepor
   validateConfiguredDrivers(config, registry, recorder);
   validateConfiguredAnswerers(config, registry, recorder);
   const caseCliTargets = await inspectCases(config, registry, recorder);
-  await inspectCliTargets([...configuredCliTargets(config), ...caseCliTargets], config, recorder);
+  await inspectCliTargets(
+    [...configuredCliTargets(config, registry), ...caseCliTargets],
+    config,
+    recorder,
+  );
   await inspectCampaigns(parsedOptions.campaigns ?? [], config, registry, recorder);
   return recorder.finish(config.rootDir);
 };

package/src/orchestrator/driver-capabilities.ts

@@ -0,0 +1,16 @@
+import type { DriverCapability, DriverPlugin } from "../core/driver.js";
+
+export const validateDriverCapabilities = (
+  plugin: DriverPlugin,
+  config: unknown,
+  required: readonly DriverCapability[] = [],
+): void => {
+  if (required.length === 0) return;
+  const capabilities = plugin.runtime?.(config).capabilities;
+  const missing = required.filter((capability) => capabilities?.[capability] !== true);
+  if (missing.length > 0) {
+    throw new Error(
+      `driver '${plugin.name}' does not declare required capabilities: ${missing.join(", ")}`,
+    );
+  }
+};

package/src/orchestrator/driver-identity.ts

@@ -0,0 +1,59 @@
+import type { DriverPlugin, DriverRuntime } from "../core/driver.js";
+import type { Registry } from "../core/registry.js";
+import { cliIdentity } from "./experiment-runtime-identity.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+
+const resolvedDriver = (
+  reference: unknown,
+  registry: Registry,
+): { plugin: DriverPlugin; config: unknown } | undefined => {
+  const name =
+    typeof reference === "string" ? reference : (reference as { name?: unknown } | null)?.name;
+  if (typeof name !== "string") return undefined;
+  const plugin = registry.getDriver(name);
+  if (!plugin) throw new Error(`unknown driver '${name}'`);
+  const raw =
+    typeof reference === "object" && reference !== null
+      ? (reference as { config?: unknown }).config
+      : undefined;
+  return { plugin, config: validatePluginConfig(plugin.configSchema, raw, `driver '${name}'`) };
+};
+
+export const referencedDriverRuntime = (reference: unknown, registry: Registry): DriverRuntime => {
+  const resolved = resolvedDriver(reference, registry);
+  return resolved?.plugin.runtime?.(resolved.config) ?? { commands: [], capabilities: {} };
+};
+
+export const experimentDriverIdentity = async (
+  reference: unknown,
+  registry: Registry,
+  rootDir: string,
+): Promise<unknown> => {
+  const resolved = resolvedDriver(reference, registry);
+  if (!resolved) return null;
+  const { plugin, config } = resolved;
+  const experiment = plugin.experiment;
+  if (!experiment)
+    throw new Error(
+      `driver '${plugin.name}' supports ordinary eval runs but does not declare controlled experiment support`,
+    );
+  if (!experiment.version.trim())
+    throw new Error(`driver '${plugin.name}' experiment version must not be empty`);
+  const prepared = validatePluginConfig(
+    plugin.configSchema,
+    experiment.prepare(config),
+    `driver '${plugin.name}' experiment configuration`,
+  );
+  const runtime = plugin.runtime?.(prepared) ?? { commands: [], capabilities: {} };
+  const behavior = await experiment.identity(config, { rootDir });
+  return {
+    driver: plugin.name,
+    version: experiment.version,
+    config: prepared,
+    capabilities: runtime.capabilities,
+    executables: await Promise.all(
+      runtime.commands.map((command) => cliIdentity(command, rootDir)),
+    ),
+    behavior,
+  };
+};

package/src/orchestrator/experiment-corpus.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import {
   bindSubjectRoot,
   type Candidate,
@@ -11,16 +12,10 @@ import {
 } from "../core/index.js";
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
+import { experimentDriverIdentity } from "./driver-identity.js";
 import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  referencedAnswererCli,
-  referencedCli,
-} from "./experiment-runtime-identity.js";
+import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
 import { loadEvalCase } from "./loader.js";
 import { resolveAnswererReference, resolveDriverReference } from "./orchestrator.js";
@@ -123,8 +118,6 @@ export const agentCorpusHash = async (
         .map((evalCase) => [evalCase.path, evalCase]),
     ).values(),
   ];
-  const caseBehaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
   const definitions = await Promise.all(
     selected.map(async (evalCase) => {
       const authored = await loadEvalCase(evalCase.path);
@@ -142,30 +135,14 @@ export const agentCorpusHash = async (
             profile.answerer ?? execution.config.defaultAnswerer,
             harness,
           );
-          const driverCli = referencedCli(driver);
           const answererCli = referencedAnswererCli(answerer);
-          const configuredRoot =
-            driverCli?.name === "claude-code" ? claudeConfigRoot(driver) : null;
-          let driverClaudeBehaviorHash: string | null = null;
-          if (configuredRoot) {
-            let hash = caseBehaviorHashes.get(configuredRoot);
-            if (!hash) {
-              hash = claudeUserBehaviorHash(configuredRoot);
-              caseBehaviorHashes.set(configuredRoot, hash);
-            }
-            driverClaudeBehaviorHash = await hash;
-          }
           return {
             profileId,
             driver,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            driverClaudeBehaviorHash,
-            configuredDriverSources: await configuredClaudeSourceHashes(
+            driverIdentity: await experimentDriverIdentity(
               driver,
+              execution.registry,
               execution.config.rootDir,
-              configuredSourceHashes,
             ),
             answerer,
             answererCliIdentity: answererCli
@@ -303,6 +280,24 @@ export const prepareCorpus = async (
     );
   }
   const owners = new Map<string, string>();
+  assertDisjointClusters(
+    splits.map((split) => ({
+      name: split.name,
+      items: selectCases(discovery.cases, split.selector),
+    })),
+  );
+  for (const constraint of campaign.objectives.constraints) {
+    if (constraint.confidence?.method !== "wilson") continue;
+    const selector = splits.find((split) => split.name === constraint.split)?.selector;
+    if (
+      selector &&
+      selectCases(discovery.cases, selector).some((evalCase) => evalCase.cluster !== undefined)
+    ) {
+      throw new Error(
+        "authored case clusters produce bounded means; use Hoeffding or percentile-bootstrap confidence instead of Wilson",
+      );
+    }
+  }
   const caseIds = new Map<string, string>();
   const evidenceOwners = new Map<
     string,

package/src/orchestrator/experiment-effects.ts

@@ -0,0 +1,61 @@
+import { z } from "zod";
+import { ConfidenceConfigSchema } from "../core/index.js";
+import type { ExperimentRun } from "./experiment-runs.js";
+import { candidateEvaluations } from "./experiment-selection.js";
+import { PairedEffectSchema, pairedEffect } from "./paired-effects.js";
+
+export const ExperimentEffectsSchema = z.array(
+  PairedEffectSchema.extend({
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    metric: z.string(),
+  }),
+);
+
+export const experimentEffects = (
+  experiment: ExperimentRun,
+): z.infer<typeof ExperimentEffectsSchema> => {
+  const candidateIds =
+    experiment.selection?.decisionCandidateIds ??
+    experiment.candidates.map((candidate) => candidate.id);
+  const baseline = candidateEvaluations(experiment, "baseline");
+  const metrics = new Set([
+    "quality.passRate",
+    experiment.evaluationKind === "grader" ? "harness.total" : "agent.total",
+    ...(experiment.evaluationKind === "agent" ? ["interrupts.ask", "toolCalls"] : []),
+    ...experiment.campaignSnapshot.objectives.constraints
+      .filter((constraint) => constraint.relativeTo)
+      .map((constraint) => constraint.metric),
+  ]);
+  return candidateIds
+    .filter((id) => id !== "baseline")
+    .flatMap((candidateId) =>
+      candidateEvaluations(experiment, candidateId).flatMap((after) =>
+        [...metrics].map((metric) => {
+          const before = baseline.find(
+            (entry) => entry.profileId === after.profileId && entry.split === after.split,
+          );
+          const constraint = experiment.campaignSnapshot.objectives.constraints.find(
+            (entry) =>
+              entry.relativeTo &&
+              entry.metric === metric &&
+              entry.split === after.split &&
+              (!entry.profile || entry.profile === after.profileId),
+          );
+          const confidence =
+            constraint?.confidence ??
+            ConfidenceConfigSchema.parse({
+              method: metric === "quality.passRate" ? "hoeffding" : "percentile-bootstrap",
+            });
+          return {
+            candidateId,
+            profileId: after.profileId,
+            split: after.split,
+            metric,
+            ...pairedEffect(before, after, metric, confidence),
+          };
+        }),
+      ),
+    );
+};

package/src/orchestrator/experiment-evaluation.ts

@@ -17,6 +17,7 @@ import {
   valueHash,
 } from "../core/index.js";
 import { materializeCandidate } from "./candidates.js";
+import { runComparisonIdentity } from "./comparison-identity.js";
 import { readContentBoundJson, writeContentBoundJson } from "./content-bound-json.js";
 import { type discoverCases, selectCases } from "./discover.js";
 import { ensureBudget, throwIfAborted } from "./experiment-budget.js";
@@ -27,6 +28,7 @@ import { average } from "./experiment-selection.js";
 import { evaluateGraderCandidate } from "./grader-experiment.js";
 import { loadEvalCase } from "./loader.js";
 import { Orchestrator, type SuiteCaseResult, type SuiteItem } from "./orchestrator.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadSuiteSummary, type SuiteSummary } from "./suite-summary.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
@@ -280,8 +282,36 @@ const agentEvaluationFromSuite = (input: {
   summary: SuiteSummary;
   results: readonly SuiteCaseResult[];
   suiteDir: string;
+  evaluationHash: string | undefined;
+  clusterId: string | undefined;
 }): CandidateEvaluation => {
   const { candidate, profileId, split, summary, results, suiteDir } = input;
+  const caseSamples = clusterCaseSamples(
+    summary.samples.map((sample) => {
+      const metadata = results.find((result) => result.caseId === sample.caseId)?.result?.metadata;
+      // Validated assertion configs contain a fresh candidate workspace path.
+      // The campaign identity binds the authored evaluator without that incidental path.
+      const identityHash =
+        metadata?.provenance && input.evaluationHash
+          ? runComparisonIdentity({
+              ...metadata,
+              provenance: { ...metadata.provenance, evaluatorConfigHash: input.evaluationHash },
+            })
+          : null;
+      return {
+        [sample.canonicalCaseId]: {
+          identityHash,
+          ...(input.clusterId ? { clusterId: input.clusterId } : {}),
+          metrics: Object.fromEntries(
+            Object.entries({
+              ...sample.metrics,
+              "quality.passRate": sample.passed === true ? 1 : 0,
+            }).map(([name, value]) => [name, { mean: value, n: 1 }]),
+          ),
+        },
+      };
+    }),
+  );
   const modelAccounting = results.map((result) =>
     modelAccountingFromMetadata(result.result?.metadata ?? null),
   );
@@ -296,6 +326,7 @@ const agentEvaluationFromSuite = (input: {
     metrics: evaluationMetrics(summary, candidate),
     metricDefinitions: evaluationMetricDefinitions(summary),
     samples: evaluationSamples(summary),
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
       (total, result) =>
@@ -362,7 +393,7 @@ const aggregateAgentUnits = (
       caseValues.set(evaluatedCaseId, values);
     }
   }
-  const samples = Object.fromEntries(
+  const ungroupedSamples = Object.fromEntries(
     [...samplesByCase].map(([name, byCase]) => [
       name,
       [...byCase.entries()]
@@ -374,6 +405,9 @@ const aggregateAgentUnits = (
     {},
     ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
   );
+  const caseSamples = clusterCaseSamples(units.map((unit) => unit.evaluation.caseSamples));
+  const grouped = groupedCaseMetricSamples(caseSamples, repetitions);
+  const samples = grouped ?? ungroupedSamples;
   return {
     candidateId: candidate.id,
     profileId,
@@ -384,13 +418,16 @@ const aggregateAgentUnits = (
       [...caseValues].map(([caseId, values]) => [caseId, average(values)]),
     ),
     metrics: Object.fromEntries(
-      [...metricValues].map(([name, values]) => [
-        name,
-        name === "subject.bytes" ? candidate.sourceBytes : average(values),
-      ]),
+      [...metricValues]
+        .filter(([name]) => !grouped || name === "subject.bytes" || Object.hasOwn(grouped, name))
+        .map(([name, values]) => [
+          name,
+          name === "subject.bytes" ? candidate.sourceBytes : average(grouped?.[name] ?? values),
+        ]),
     ),
     metricDefinitions,
     samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
     unpricedModelCalls: units.reduce(
@@ -461,6 +498,7 @@ export const evaluateAgentCandidate = async (options: {
   const defaultDriver = profile.driver ?? execution.config.defaultDriver;
   const registry = execution.registry.withAssertion(createRubricAssertion(harness));
   const orchestrator = new Orchestrator({
+    environment: profile.environment ?? execution.config.environment,
     registry,
     defaultAnswerer: profile.answerer ?? execution.config.defaultAnswerer,
     ...(defaultDriver ? { defaultDriver } : {}),
@@ -629,6 +667,8 @@ export const evaluateAgentCandidate = async (options: {
             summary,
             results: suite.results,
             suiteDir: suite.suiteDir,
+            evaluationHash: experiment.manifest.evaluationHash,
+            clusterId: evalCase.cluster,
           });
         } finally {
           await rm(unitRoot, { recursive: true, force: true });