@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/profile.ts

@@ -0,0 +1,24 @@
+import {
+  mergeAnswererReference,
+  mergeDriverReference,
+  withDriverDefaults,
+} from "../core/reference-defaults.js";
+import type { ExperimentProfile, ResolvedLuxConfig } from "./config.js";
+
+/** Used by ordinary runs, doctor, and experiment execution and identity alike. */
+export const resolveProfile = (
+  config: Pick<
+    ResolvedLuxConfig,
+    "defaultDriver" | "defaultAnswerer" | "harness" | "environment" | "driverDefaults"
+  >,
+  profile: ExperimentProfile = {},
+): ExperimentProfile => ({
+  ...profile,
+  driver: withDriverDefaults(
+    mergeDriverReference(config.defaultDriver, profile.driver),
+    config.driverDefaults,
+  ),
+  answerer: mergeAnswererReference(config.defaultAnswerer, profile.answerer),
+  harness: profile.harness ?? config.harness,
+  environment: profile.environment ?? config.environment,
+});

package/src/orchestrator/provenance.ts

@@ -1,6 +1,8 @@
 import { existsSync } from "node:fs";
 import { lstat, readFile, readlink } from "node:fs/promises";
+import { availableParallelism, cpus, release, totalmem } from "node:os";
 import { dirname, join, parse, relative, resolve, sep } from "node:path";
+import type { EnvironmentDeclaration } from "../core/environment.js";
 import type { RunProvenance } from "../core/index.js";
 import { contentHash, exec, valueHash } from "../core/index.js";
 
@@ -113,13 +115,30 @@ export const runtimeIdentity = (): RunProvenance["runtime"] => ({
   arch: process.arch,
   node: process.version,
   ...(process.versions.bun ? { bun: process.versions.bun } : {}),
+  host: {
+    osRelease: release(),
+    cpuModels: [...new Set(cpus().map((cpu) => cpu.model))].sort(),
+    logicalCpuCount: cpus().length,
+    availableParallelism: availableParallelism(),
+    totalMemoryBytes: totalmem(),
+  },
 });
 
-export const runProvenance = async (casePath: string | null): Promise<RunProvenance> => {
+export const runProvenance = async (
+  casePath: string | null,
+  options: {
+    environment?: EnvironmentDeclaration | undefined;
+    concurrency?: number | undefined;
+  } = {},
+): Promise<RunProvenance> => {
   const runtime = runtimeIdentity();
-  if (!casePath) return { runtime };
+  const environment = {
+    ...(options.environment ? { declared: options.environment } : {}),
+    concurrency: options.concurrency ?? 1,
+  };
+  if (!casePath) return { runtime, environment };
   const root = gitRootAbove(casePath);
-  if (!root) return { runtime, source: { casePath: resolve(casePath) } };
+  if (!root) return { runtime, environment, source: { casePath: resolve(casePath) } };
   let git = repositories.get(root);
   if (!git) {
     git = inspectRepository(root);
@@ -128,6 +147,7 @@ export const runProvenance = async (casePath: string | null): Promise<RunProvena
   const state = await git;
   return {
     runtime,
+    environment,
     source: {
       casePath: portablePath(relative(root, resolve(casePath))),
       ...(state ? { git: state } : {}),

package/src/orchestrator/rubric-loader.ts

@@ -9,7 +9,7 @@ const CANDIDATE_FORBIDDEN_FIELDS = [
   "provider",
   "model",
   "reasoningEffort",
-  "timeoutMs",
+  "timeout",
   "promptOverride",
 ] as const;
 const RawRubricDefinitionSchema = z.looseObject({

package/src/orchestrator/run-dir.ts

@@ -154,6 +154,8 @@ export type OpenRunDirOptions = {
   runsRoot: string;
   caseId: string;
   driverName: string;
+  /** Files the run under the permutation instead of the bare driver name. */
+  permutationId?: string | undefined;
   startedAt?: Date | undefined;
   /** Tap on the event stream, called for every appended event. Must not throw. */
   onEvent?: ((event: RunEvent) => void) | undefined;
@@ -164,7 +166,12 @@ export type OpenRunDirOptions = {
 export const openRunDir = async (options: OpenRunDirOptions): Promise<OpenedRunDir> => {
   const ts = isoTimestampSafe(options.startedAt ?? new Date());
   const slug = slugifyCaseId(options.caseId);
-  const basePath = join(options.runsRoot, slug, slugifyCaseId(options.driverName), ts);
+  const basePath = join(
+    options.runsRoot,
+    slug,
+    slugifyCaseId(options.permutationId ?? options.driverName),
+    ts,
+  );
   await mkdir(dirname(basePath), { recursive: true });
   // Same case+driver+millisecond must not share a dir — their events.jsonl
   // streams (opened with flags "a") would interleave silently. A bare mkdir

package/src/orchestrator/run-execution.ts

@@ -1,6 +1,7 @@
 import { cp } from "node:fs/promises";
 import { dirname, isAbsolute, resolve } from "node:path";
 import { z } from "zod";
+import type { EnvironmentDeclaration } from "../core/environment.js";
 import {
   type AgentDriver,
   type Answerer,
@@ -10,6 +11,8 @@ import {
   AnswererModelCallAccountingSchema,
   type AnswerHistoryEntry,
   type Artifact,
+  actionTiming,
+  bindRunExecutionWorkspace,
   type EvalCase,
   emptyUsage,
   type FixturesRef,
@@ -18,6 +21,7 @@ import {
   guardRunObserver,
   inspectRunArtifact,
   type LifecycleStage,
+  normalizeDriverMetrics,
   type OperationalFailure,
   type PhaseDurations,
   type ReadyState,
@@ -28,12 +32,14 @@ import {
   type RunMetadata,
   type RunProvenance,
   type RunUsage,
+  summarizeAgentActivity,
   summarizeToolOutcomes,
   sumUsage,
   type TokenUsage,
   TokenUsageSchema,
   valueHash,
 } from "../core/index.js";
+import { identifyArtifactOrigins } from "./artifact-origin.js";
 import {
   createDriverSession,
   type DriverSession,
@@ -49,6 +55,7 @@ import {
   type RunFailurePhase,
   type RunLifecycleEffects,
 } from "./run-lifecycle.js";
+import { RunWorkspace } from "./run-workspace.js";
 import { isRunTraceLimit, type RunTraceLimitFailure } from "./trace-limits.js";
 
 /** Sentinel `casePath` for a pre-loaded case that has no file behind it. */
@@ -57,6 +64,7 @@ export const INLINE_CASE_PATH = "<inline>";
 type RunDirCarrier = { luxRunDir?: string };
 type InterruptCounts = { ask: number; approve: number; custom: number };
 export type RunWithPluginsArgs = {
+  environment?: EnvironmentDeclaration | undefined;
   request: RunRequest;
   casePath: string;
   authoredCase: EvalCase;
@@ -64,6 +72,7 @@ export type RunWithPluginsArgs = {
   validatedAssertionConfigs: readonly unknown[];
   caseId: string;
   driverName: string;
+  permutationId?: string | undefined;
   answererName: string;
   answererConfig: unknown;
   driverConfigHash: string;
@@ -108,7 +117,9 @@ const readAnswererUsage = (answerer: Answerer): TokenUsage =>
 const stageFixtures = async (
   ref: FixturesRef | undefined,
   options: { fixturesRoot?: string; casePath: string; artifactsDir: string },
-): Promise<{ fixtures: ResolvedFixtures; hash: string } | undefined> => {
+): Promise<
+  { fixtures: ResolvedFixtures; hash: string; fileHashes: ReadonlyMap<string, string> } | undefined
+> => {
   if (!ref) return undefined;
   const path = typeof ref === "string" ? ref : ref.path;
   if (!isAbsolute(path) && !options.fixturesRoot && options.casePath === INLINE_CASE_PATH) {
@@ -123,7 +134,11 @@ const stageFixtures = async (
   if (staged.hash !== source.hash) {
     throw new Error(`fixtures: tree changed while staging: ${source.rootDir}`);
   }
-  return { fixtures: { rootDir: source.rootDir }, hash: staged.hash };
+  return {
+    fixtures: { rootDir: source.rootDir },
+    hash: staged.hash,
+    fileHashes: staged.fileHashes,
+  };
 };
 
 export const runDirOfError = (error: unknown): string | null =>
@@ -170,8 +185,17 @@ const SummaryModelAccountingSchema = z.object({
   unpricedModelCalls: z.number().int().nonnegative().optional(),
 });
 
-const usageFromSummary = (summary: unknown): TokenUsage | null =>
-  SummaryUsageSchema.safeParse(summary).data?.usage ?? null;
+const usageFromSummary = (summary: unknown): TokenUsage | null => {
+  if (!summary || typeof summary !== "object") return null;
+  const raw = (summary as { usage?: unknown }).usage;
+  if (!raw || typeof raw !== "object") return null;
+  const counts = raw as Record<string, unknown>;
+  const hasCount = ["input", "output", "cacheRead", "cacheCreation"].some((key) => {
+    const value = counts[key];
+    return typeof value === "number" && Number.isInteger(value) && value >= 0;
+  });
+  return hasCount ? (SummaryUsageSchema.safeParse(summary).data?.usage ?? null) : null;
+};
 
 const costFromSummary = (summary: unknown): number | undefined => {
   if (!summary || typeof summary !== "object") return undefined;
@@ -183,6 +207,11 @@ const modelAccountingFromSummary = (
 ): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
   SummaryModelAccountingSchema.safeParse(summary).data ?? null;
 
+const metricsFromSummary = (summary: unknown) => {
+  if (!summary || typeof summary !== "object") return {};
+  return normalizeDriverMetrics((summary as { metrics?: unknown }).metrics);
+};
+
 const artifactMeasurements = async (
   artifacts: readonly Artifact[],
   runDir: string,
@@ -287,12 +316,14 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
   private loop: DriverSession | undefined;
   private ready: ReadyState | undefined;
   private fixturesHash: string | undefined;
+  private fixtureFileHashes: ReadonlyMap<string, string> = new Map();
   private startAttempted = false;
   private loopOutcome: DriverSessionOutcome | undefined;
   private collection: Collection = { artifacts: [] };
   private metadata: RunMetadata | undefined;
   private usage: RunUsage | undefined;
   private run: Run | undefined;
+  private workspace: RunWorkspace | undefined;
 
   private constructor(
     args: RunWithPluginsArgs,
@@ -312,12 +343,14 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const startedAt = args.request.startedAt ?? new Date();
     const provenance = await runProvenance(
       args.casePath === INLINE_CASE_PATH ? null : args.casePath,
+      { environment: args.environment, concurrency: args.request.suiteConcurrency },
     );
     const observer = guardRunObserver(args.request.observer);
     const runDir = await openRunDir({
       runsRoot: resolve(args.request.runsRoot),
       caseId: args.caseId,
       driverName: args.driverName,
+      permutationId: args.permutationId,
       startedAt,
       onEvent: observer?.event,
     });
@@ -347,12 +380,14 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const { request, evalCase, casePath, driver } = this.args;
     this.lifecycle("run-loop");
     if (request.abortSignal?.aborted) return;
+    this.workspace = await RunWorkspace.create();
     const staged = await stageFixtures(evalCase.fixtures, {
       ...(request.fixturesRoot ? { fixturesRoot: request.fixturesRoot } : {}),
       casePath,
-      artifactsDir: this.runDir.artifactsDir,
+      artifactsDir: this.workspace.path,
     });
     this.fixturesHash = staged?.hash;
+    this.fixtureFileHashes = staged?.fileHashes ?? new Map();
     if (request.abortSignal?.aborted) return;
 
     this.lifecycle("start-driver");
@@ -365,7 +400,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       ...(this.args.fixtureValues.length > 0 ? { fixtureValues: this.args.fixtureValues } : {}),
       runsRoot: resolve(request.runsRoot),
       runDir: this.runDir.path,
-      artifactsDir: this.runDir.artifactsDir,
+      artifactsDir: this.workspace.path,
       abortSignal: request.abortSignal ?? new AbortController().signal,
     });
     this.append({ type: "ready", ...this.ready });
@@ -390,6 +425,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     metadata.exitReason = "error";
     metadata.operationalFailure = { phase: operationalPhase, reason };
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.writeJson("run.json", metadata);
     await this.args.request.recordPreGradingAccounting?.(metadata);
     await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
@@ -407,9 +443,12 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     this.lifecycle("collect-end");
     await this.runDir.events.drain();
     const completed = await this.completedAgentRun();
+    completed.metadata.actionTiming = actionTiming(this.runDir.recorded);
     this.metadata = completed.metadata;
     this.usage = completed.usage;
     this.run = completed.run;
+    const executionWorkspace = await this.args.driver.verificationWorkspace?.();
+    if (executionWorkspace) bindRunExecutionWorkspace(this.run, executionWorkspace);
     await writeFrozenCase(this.runDir.path, this.args.authoredCase, this.args.casePath);
     await this.runDir.writeJson("run.json", completed.metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -430,6 +469,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
     );
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.events.drain();
     await this.runDir.writeJson("run.json", metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -438,6 +478,12 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
   }
 
   async close(primaryError: unknown | null): Promise<void> {
+    if (this.workspace) {
+      await closeQuietly(async () => {
+        await this.args.driver.close();
+        await this.workspace?.archive(this.runDir.artifactsDir);
+      }, `run workspace ${this.workspace.path}`);
+    }
     if (primaryError instanceof Error) {
       (primaryError as Error & RunDirCarrier).luxRunDir = this.runDir.path;
     }
@@ -506,6 +552,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
   private fallbackFailureMetadata(): RunMetadata {
     const failedAt = new Date();
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
+    const ready = this.ready ?? { agentId: "start-failed" };
     return {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -514,8 +561,9 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: failedAt.toISOString(),
       endedAt: failedAt.toISOString(),
       exitReason: "error",
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "start-failed" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
+      agentUsageAvailability: "unavailable",
       usage: {
         agent: emptyUsage(),
         answerer: readAnswererUsage(this.args.answerer),
@@ -530,6 +578,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: { count: this.collection.artifacts.length },
       provenance: this.runProvenance(),
     };
@@ -539,7 +588,16 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     if (!this.ready) return { artifacts: [] };
     try {
       await this.args.driver.quiesce?.();
-      return { artifacts: await this.args.driver.collect() };
+      const artifacts = await this.args.driver.collect();
+      const archived =
+        (await this.workspace?.archive(this.runDir.artifactsDir, artifacts)) ?? artifacts;
+      return {
+        artifacts: await identifyArtifactOrigins(
+          this.runDir.path,
+          archived,
+          this.fixtureFileHashes,
+        ),
+      };
     } catch (error) {
       const reason = errorMessage(error);
       this.runDir.events.append({
@@ -563,14 +621,17 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const driverMetrics = metricsFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
+    const agentUsage = usageFromSummary(summary);
     const usage: RunUsage = {
-      agent: usageFromSummary(summary) ?? emptyUsage(),
+      agent: agentUsage ?? emptyUsage(),
       answerer: readAnswererUsage(this.args.answerer),
       grading: emptyUsage(),
     };
+    const ready = this.ready ?? { agentId: "not-started" };
     const metadata: RunMetadata = {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -579,9 +640,10 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: agentEndedAt.toISOString(),
       endedAt: agentEndedAt.toISOString(),
       exitReason,
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "not-started" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage,
+      agentUsageAvailability: agentUsage === null ? "unavailable" : "reported",
       modelAccounting: {
         agentCalls,
         answererCalls: answererAccounting.calls,
@@ -593,10 +655,12 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: await artifactMeasurements(this.collection.artifacts, this.runDir.path),
       ...(operationalFailure ? { operationalFailure } : {}),
       provenance: this.runProvenance(),
       ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+      ...(Object.keys(driverMetrics).length > 0 ? { driverMetrics } : {}),
     };
     return {
       metadata,
@@ -621,6 +685,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       ...(this.fixturesHash ? { fixturesHash: this.fixturesHash } : {}),
       ...(request.subjectHash ? { subjectHash: request.subjectHash } : {}),
       ...(request.profileId ? { profileId: request.profileId } : {}),
+      ...(this.args.permutationId ? { permutationId: this.args.permutationId } : {}),
       ...(request.experimentId ? { experimentId: request.experimentId } : {}),
       ...(request.variantId ? { variantId: request.variantId } : {}),
       ...(request.candidateId ? { candidateId: request.candidateId } : {}),

package/src/orchestrator/run-lifecycle.ts

@@ -23,6 +23,7 @@ type RunLifecycleContext = {
   phase: RunFailurePhase;
   result: unknown;
   error: unknown;
+  failed: boolean;
 };
 type RunLifecycleInput = { effects: ErasedEffects };
 type RunLifecycleOutcome =
@@ -56,7 +57,7 @@ const runLifecycleMachine = setup({
   actions: {
     captureError: assign(({ context, event }) => {
       const error = "error" in event ? event.error : new Error("run lifecycle failed");
-      return { error, phase: isRunTraceLimit(error) ? "trace" : context.phase };
+      return { error, failed: true, phase: isRunTraceLimit(error) ? "trace" : context.phase };
     }),
     captureResult: assign({
       result: ({ event }) => ("output" in event ? event.output : undefined),
@@ -73,6 +74,7 @@ const runLifecycleMachine = setup({
     phase: "setup",
     result: undefined,
     error: undefined,
+    failed: false,
   }),
   initial: "preparing",
   states: {
@@ -153,7 +155,7 @@ const runLifecycleMachine = setup({
     failed: { type: "final" },
   },
   output: ({ context }) => {
-    if (context.error !== undefined) {
+    if (context.failed) {
       return { status: "failed", error: context.error };
     }
     return { status: "succeeded", result: context.result };

package/src/orchestrator/run-workspace.ts

@@ -0,0 +1,62 @@
+import { cp, lstat, mkdir, mkdtemp, realpath, rename, rm, rmdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import type { Artifact } from "../core/index.js";
+
+/** Keep package-manager discovery and repository ignore rules outside subject workspaces. */
+export class RunWorkspace {
+  private archived = false;
+  readonly path: string;
+
+  private constructor(path: string) {
+    this.path = path;
+  }
+
+  static async create(): Promise<RunWorkspace> {
+    const root = join(tmpdir(), `lux-workspaces-${process.getuid?.() ?? "user"}`);
+    await mkdir(root, { recursive: true, mode: 0o700 });
+    const metadata = await lstat(root);
+    if (
+      !metadata.isDirectory() ||
+      metadata.isSymbolicLink() ||
+      (process.getuid && metadata.uid !== process.getuid())
+    ) {
+      throw new Error(
+        `Lux workspace root must be a real directory owned by the current user: ${root}`,
+      );
+    }
+    return new RunWorkspace(await realpath(await mkdtemp(join(root, "run-"))));
+  }
+
+  async archive(destination: string, artifacts: Artifact[] = []): Promise<Artifact[]> {
+    if (!this.archived) {
+      const metadata = await lstat(this.path);
+      if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
+        throw new Error(`run workspace must remain a real directory: ${this.path}`);
+      }
+      await rmdir(destination).catch((error: NodeJS.ErrnoException) => {
+        if (error.code !== "ENOENT") throw error;
+      });
+      try {
+        await rename(this.path, destination);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "EXDEV") throw error;
+        await cp(this.path, destination, { recursive: true, verbatimSymlinks: true });
+        await rm(this.path, { recursive: true });
+      }
+      this.archived = true;
+    }
+    return artifacts.map((artifact) => {
+      if (!artifact.root) return artifact;
+      const path = relative(this.path, resolve(artifact.root));
+      if (path === "") {
+        const { root: _root, ...captured } = artifact;
+        return captured;
+      }
+      if (path === ".." || path.startsWith(`..${sep}`) || isAbsolute(path)) {
+        return artifact;
+      }
+      return { ...artifact, root: join(destination, path) };
+    });
+  }
+}

package/src/orchestrator/suite-compare.ts

@@ -5,6 +5,7 @@ import {
   MetricSummarySchema,
   type SuiteSample,
   type SuiteSummary,
+  suiteGroupKey,
   summarizeNumbers,
 } from "./suite-summary.js";
 
@@ -18,6 +19,7 @@ const ComparedOutcomeSchema = z
 export const SuiteCaseComparisonSchema = z
   .object({
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     before: ComparedOutcomeSchema.nullable(),
     after: ComparedOutcomeSchema.nullable(),
     passRateDelta: z.number().nullable(),
@@ -40,6 +42,7 @@ export const SuiteComparisonSchema = z
       z
         .object({
           canonicalCaseId: z.string(),
+          permutationId: z.exactOptional(z.string()),
           repetition: z.number().int().positive(),
           before: z.string().nullable(),
           after: z.string().nullable(),
@@ -54,9 +57,10 @@ export type SuiteComparison = z.infer<typeof SuiteComparisonSchema>;
 const samplesByCase = (summary: SuiteSummary): Map<string, SuiteSample[]> => {
   const out = new Map<string, SuiteSample[]>();
   for (const sample of summary.samples) {
-    const entries = out.get(sample.canonicalCaseId) ?? [];
+    const key = suiteGroupKey(sample);
+    const entries = out.get(key) ?? [];
     entries.push(sample);
-    out.set(sample.canonicalCaseId, entries);
+    out.set(key, entries);
   }
   return out;
 };
@@ -70,10 +74,19 @@ const pairedMetricDeltas = (
     const a = before[index];
     const b = after[index];
     if (!a || !b) continue;
-    const names = new Set([...Object.keys(a.metrics), ...Object.keys(b.metrics)]);
+    const am: Record<string, number> = {
+      ...a.metrics,
+      ...(a.score !== undefined ? { score: a.score } : {}),
+    };
+    const bm: Record<string, number> = {
+      ...b.metrics,
+      ...(b.score !== undefined ? { score: b.score } : {}),
+    };
+    const names = new Set([...Object.keys(am), ...Object.keys(bm)]);
     for (const name of names) {
-      const av = a.metrics[name];
-      const bv = b.metrics[name];
+      if (name === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis)) continue;
+      const av = am[name];
+      const bv = bm[name];
       if (av === undefined || bv === undefined) continue;
       const values = deltas.get(name) ?? [];
       values.push(bv - av);
@@ -95,14 +108,23 @@ export const compareSuiteSummaries = (
   options: { passRateTolerance?: number } = {},
 ): SuiteComparison => {
   const passRateTolerance = options.passRateTolerance ?? 0;
-  const aCases = new Map(a.cases.map((entry) => [entry.canonicalCaseId, entry]));
-  const bCases = new Map(b.cases.map((entry) => [entry.canonicalCaseId, entry]));
+  const aCases = new Map(a.cases.map((entry) => [suiteGroupKey(entry), entry]));
+  const bCases = new Map(b.cases.map((entry) => [suiteGroupKey(entry), entry]));
   const aSamples = samplesByCase(a);
   const bSamples = samplesByCase(b);
-  const ids = new Set([...aCases.keys(), ...bCases.keys()]);
-  const identityMismatches = [...ids].sort().flatMap((canonicalCaseId) => {
-    const before = aSamples.get(canonicalCaseId) ?? [];
-    const after = bSamples.get(canonicalCaseId) ?? [];
+  const groups = new Map(
+    [...a.cases, ...b.cases].map((entry) => [
+      suiteGroupKey(entry),
+      {
+        canonicalCaseId: entry.canonicalCaseId,
+        ...(entry.permutationId ? { permutationId: entry.permutationId } : {}),
+      },
+    ]),
+  );
+  const keys = [...groups.keys()].sort();
+  const identityMismatches = keys.flatMap((key) => {
+    const before = aSamples.get(key) ?? [];
+    const after = bSamples.get(key) ?? [];
     return Array.from({ length: Math.max(before.length, after.length) }, (_, index) => {
       const beforeHash = before[index]?.identityHash ?? null;
       const afterHash = after[index]?.identityHash ?? null;
@@ -110,7 +132,7 @@ export const compareSuiteSummaries = (
         ? []
         : [
             {
-              canonicalCaseId,
+              ...groups.get(key),
               repetition: index + 1,
               before: beforeHash,
               after: afterHash,
@@ -118,9 +140,9 @@ export const compareSuiteSummaries = (
           ];
     }).flat();
   });
-  const cases = [...ids].sort().map((canonicalCaseId) => {
-    const before = aCases.get(canonicalCaseId);
-    const after = bCases.get(canonicalCaseId);
+  const cases = keys.map((key) => {
+    const before = aCases.get(key);
+    const after = bCases.get(key);
     let passRateDelta: number | null = null;
     if (before && after) {
       passRateDelta = after.passRate - before.passRate;
@@ -128,16 +150,13 @@ export const compareSuiteSummaries = (
       passRateDelta = -before.passRate;
     }
     return {
-      canonicalCaseId,
+      ...groups.get(key),
       before: before ? { attempts: before.attempts, passRate: before.passRate } : null,
       after: after ? { attempts: after.attempts, passRate: after.passRate } : null,
       passRateDelta,
       regressed:
         Boolean(before) && (!after || after.passRate + passRateTolerance < (before?.passRate ?? 0)),
-      pairedMetricDeltas: pairedMetricDeltas(
-        aSamples.get(canonicalCaseId) ?? [],
-        bSamples.get(canonicalCaseId) ?? [],
-      ),
+      pairedMetricDeltas: pairedMetricDeltas(aSamples.get(key) ?? [], bSamples.get(key) ?? []),
     };
   });
   return SuiteComparisonSchema.parse({
@@ -182,13 +201,18 @@ export const formatSuiteComparisonText = (comparison: SuiteComparison): string =
   if (comparison.identityMismatches.length > 0) {
     lines.push(`  identity mismatches: ${comparison.identityMismatches.length}`);
     for (const mismatch of comparison.identityMismatches) {
-      lines.push(`    ! ${mismatch.canonicalCaseId} repetition ${mismatch.repetition}`);
+      lines.push(`    ! ${suiteGroupKey(mismatch)} repetition ${mismatch.repetition}`);
     }
   }
   for (const entry of comparison.cases) {
-    if (!entry.regressed) continue;
+    if (entry.regressed) {
+      lines.push(
+        `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      );
+    }
+    const score = entry.pairedMetricDeltas.score;
     lines.push(
-      `    ↓ ${entry.canonicalCaseId}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      `    ${suiteGroupKey(entry)} score change: ${score ? `${score.mean >= 0 ? "+" : ""}${score.mean} (${score.n} paired)` : "unavailable"}`,
     );
   }
   return lines.join("\n");