@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/experiment-report.ts

@@ -2,6 +2,7 @@ import { basename } from "node:path";
 import { z } from "zod";
 import { valueHash } from "../core/index.js";
 import { deriveExperimentSelection, rankCandidates } from "./experiment.js";
+import { ExperimentEffectsSchema, experimentEffects } from "./experiment-effects.js";
 import {
   type ExperimentRun,
   ExperimentRunSchema,
@@ -24,6 +25,7 @@ export const ExperimentDecisionReportSchema = z
     status: z.enum(["running", "passed", "failed", "aborted"]),
     outcome: z.enum(["promote", "retain", "reject", "error", "incomplete"]),
     error: z.string().nullable(),
+    effects: z.exactOptional(ExperimentEffectsSchema),
     candidateAndAccountingVerified: z.boolean(),
     championCandidateId: z.string().nullable(),
     ranking: z.array(z.string()),
@@ -39,6 +41,7 @@ export const ExperimentDecisionReportSchema = z
       z
         .object({
           metric: z.string(),
+          relativeTo: z.exactOptional(z.literal("baseline")),
           profile: z.string().nullable(),
           split: ExperimentReportSplitSchema,
           passed: z.boolean(),
@@ -596,6 +599,7 @@ const projectExperimentDecisionReport = (
     evaluationKind: experiment.evaluationKind,
     status: experiment.status,
     error: experiment.errorMessage ?? null,
+    effects: experimentEffects(experiment),
     candidateAndAccountingVerified,
     outcome: reportOutcome(experiment, championId),
     championCandidateId: championId,
@@ -610,6 +614,7 @@ const projectExperimentDecisionReport = (
       .filter((constraint) => constraint.candidateId === targetId)
       .map((constraint) => ({
         metric: constraint.metric,
+        ...(constraint.relativeTo ? { relativeTo: constraint.relativeTo } : {}),
         profile: constraint.profile ?? null,
         split: constraint.split,
         passed: constraint.passed,
@@ -733,7 +738,7 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       "| --- | --- | --- | --- | ---: | --- | --- |",
       ...report.gates.map(
         (gate) =>
-          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
+          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)}${gate.relativeTo ? " Δ baseline" : ""} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
       ),
     );
   }
@@ -763,6 +768,21 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       ),
     );
   }
+  if (report.effects?.length) {
+    lines.push(
+      "",
+      "## Paired case effects",
+      "",
+      "Differences are candidate minus baseline; repetitions are averaged within cases and authored clusters receive equal weight.",
+      "",
+      "| Candidate | Profile | Split | Metric | Mean difference | Cases | Clusters | Confidence | Note |",
+      "| --- | --- | --- | --- | ---: | ---: | ---: | --- | --- |",
+      ...report.effects.map(
+        (effect) =>
+          `| ${markdown(effect.candidateId)} | ${markdown(effect.profileId)} | ${effect.split} | ${markdown(effect.metric)} | ${effect.meanDelta === null ? "—" : signed(effect.meanDelta)} | ${effect.caseCount ?? effect.caseIds.length} | ${effect.n} | ${markdown(intervalText(effect.interval ?? undefined) ?? "—")} | ${markdown(effect.note ?? "—")} |`,
+      ),
+    );
+  }
   lines.push(
     "",
     "## Source and spend",

package/src/orchestrator/experiment-runs.ts

@@ -86,6 +86,7 @@ export const ExperimentSelectionSchema = z.object({
     z.object({
       candidateId: CandidateIdSchema,
       metric: z.string(),
+      relativeTo: z.exactOptional(z.literal("baseline")),
       profile: z.exactOptional(z.string()),
       split: z.enum(["train", "validation", "test"]),
       values: z.array(z.number()),
@@ -738,6 +739,7 @@ export const resumeExperimentRun = async (path: string): Promise<OpenedExperimen
 
 export const listExperimentRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<ExperimentRun & { experimentDir: string }>> => {
   const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
   if (!existsSync(root)) return [];
@@ -745,10 +747,15 @@ export const listExperimentRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const runs = await Promise.all(
-    directories.map(async (experimentDir) => ({
-      experimentDir,
-      experiment: await loadExperimentRun(experimentDir),
-    })),
+    directories.map(async (experimentDir) => {
+      try {
+        return { experimentDir, experiment: await loadExperimentRun(experimentDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(experimentDir, error);
+        return { experimentDir, experiment: null };
+      }
+    }),
   );
   return runs
     .flatMap(({ experimentDir, experiment }) =>

package/src/orchestrator/experiment-runtime-identity.ts

@@ -1,61 +1,14 @@
 import { createHash } from "node:crypto";
-import { access, constants, lstat, open, readFile, realpath } from "node:fs/promises";
-import { homedir } from "node:os";
+import { access, constants, lstat, open, realpath } from "node:fs/promises";
 import { delimiter, isAbsolute, join, resolve } from "node:path";
 import { z } from "zod";
-import { contentHash, discoverCliVersion, valueHash } from "../core/index.js";
-import { walkAuthoredFiles } from "./fs-read.js";
-import { boundExperimentDriverConfig } from "./orchestrator.js";
+import { discoverCliVersion, valueHash } from "../core/index.js";
 
 export { LUX_VERSION } from "./package-version.js";
 
-const ClaudeSettingsFileSchema = z.looseObject({
-  enabledPlugins: z.record(z.string(), z.boolean()).optional(),
-});
-const ClaudeInstalledPluginRecordSchema = z.looseObject({
-  installPath: z.string(),
-});
-const ClaudeInstalledPluginsFileSchema = z.looseObject({
-  plugins: z.record(z.string(), z.array(ClaudeInstalledPluginRecordSchema)),
-});
-
-export const referencedCli = (
-  reference: unknown,
-): { name: string; command: string } | undefined => {
-  let name: string | undefined;
-  if (typeof reference === "string") {
-    name = reference;
-  } else if (
-    typeof reference === "object" &&
-    reference !== null &&
-    typeof (reference as { name?: unknown }).name === "string"
-  ) {
-    name = (reference as { name: string }).name;
-  }
-  const config =
-    typeof reference === "object" && reference !== null
-      ? (reference as { config?: unknown }).config
-      : undefined;
-  if (name === "subprocess") {
-    if (
-      typeof config !== "object" ||
-      config === null ||
-      typeof (config as { command?: unknown }).command !== "string"
-    ) {
-      return undefined;
-    }
-    return { name, command: (config as { command: string }).command };
-  }
-  if (name !== "claude-code" && name !== "codex") return undefined;
-  let command = name === "codex" ? "codex" : "claude";
-  if (
-    typeof config === "object" &&
-    config !== null &&
-    typeof (config as { command?: unknown }).command === "string"
-  ) {
-    command = (config as { command: string }).command;
-  }
-  return { name, command };
+const driverConfigRecord = (reference: unknown): Record<string, unknown> => {
+  const config = (reference as { config?: unknown } | null)?.config;
+  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
 };
 
 export const referencedAnswererCli = (
@@ -261,197 +214,6 @@ export const cliIdentity = async (command: string, baseRoot: string): Promise<Cl
   });
 };
 
-const optionalRegularFileHash = async (path: string): Promise<string | null> => {
-  try {
-    const info = await lstat(path);
-    if (info.isSymbolicLink() || !info.isFile()) {
-      throw new Error(`behavior-affecting settings must be a regular file: ${path}`);
-    }
-    return contentHash(await readFile(path));
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
-    throw error;
-  }
-};
-
-const driverConfigRecord = (driver: unknown): Record<string, unknown> => {
-  if (!driver || typeof driver !== "object") return {};
-  const config = (driver as { config?: unknown }).config;
-  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
-};
-
-export const claudeConfigRoot = (driver: unknown): string => {
-  const config = driverConfigRecord(driver);
-  const env =
-    config.env && typeof config.env === "object"
-      ? (config.env as Record<string, unknown>)
-      : undefined;
-  const configured = env?.CLAUDE_CONFIG_DIR;
-  const root =
-    typeof configured === "string"
-      ? configured
-      : (process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"));
-  if (!isAbsolute(root)) {
-    throw new Error("CLAUDE_CONFIG_DIR must be absolute for reproducible experiment identity");
-  }
-  return resolve(root);
-};
-
-const hashBehaviorDirectory = async (
-  root: string,
-  label: string,
-): Promise<Array<{ path: string; hash: string }>> => {
-  try {
-    const info = await lstat(root);
-    if (info.isSymbolicLink() || !info.isDirectory()) {
-      throw new Error(`behavior-affecting Claude source must be a real directory: ${root}`);
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
-    throw error;
-  }
-  const paths = await walkAuthoredFiles(root);
-  if (paths.length > 5_000) {
-    throw new Error(`behavior-affecting Claude source '${label}' exceeds 5000 files`);
-  }
-  let bytes = 0;
-  const hashes: Array<{ path: string; hash: string }> = [];
-  for (const path of paths) {
-    const content = await readFile(join(root, path));
-    bytes += content.length;
-    if (bytes > 256 * 1024 * 1024) {
-      throw new Error(`behavior-affecting Claude source '${label}' exceeds 256 MiB`);
-    }
-    hashes.push({ path: `${label}/${path}`, hash: contentHash(content) });
-  }
-  return hashes;
-};
-
-/**
- * Hash every intentionally loaded user behavior source without persisting its
- * contents. This covers user instructions/rules/output styles and the exact
- * installed contents of plugins enabled by settings.json.
- */
-export const claudeUserBehaviorHash = async (configRoot: string): Promise<string | null> => {
-  const settingsPath = join(configRoot, "settings.json");
-  const settingsHash = await optionalRegularFileHash(settingsPath);
-  const sources: Array<{ path: string; hash: string }> = [];
-  if (settingsHash) sources.push({ path: "settings.json", hash: settingsHash });
-  const claudeMdHash = await optionalRegularFileHash(join(configRoot, "CLAUDE.md"));
-  if (claudeMdHash) sources.push({ path: "CLAUDE.md", hash: claudeMdHash });
-  for (const directory of ["rules", "output-styles", "agents", "commands", "skills"]) {
-    sources.push(...(await hashBehaviorDirectory(join(configRoot, directory), directory)));
-  }
-
-  let settings: unknown;
-  try {
-    const contents = await readFile(settingsPath, "utf8");
-    try {
-      settings = JSON.parse(contents);
-    } catch {
-      throw new Error("Claude settings.json contains invalid JSON");
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      settings = undefined;
-    } else {
-      throw error;
-    }
-  }
-  const parsedSettings = ClaudeSettingsFileSchema.safeParse(settings);
-  if (settings !== undefined && !parsedSettings.success) {
-    throw new Error("Claude settings.json has an invalid enabledPlugins map");
-  }
-  const enabledPlugins = parsedSettings.success ? parsedSettings.data.enabledPlugins : undefined;
-  const enabledIds = Object.entries(enabledPlugins ?? {})
-    .filter(([, value]) => value === true)
-    .map(([id]) => id)
-    .sort();
-  if (enabledIds.length > 0) {
-    const installedPath = join(configRoot, "plugins", "installed_plugins.json");
-    const installedHash = await optionalRegularFileHash(installedPath);
-    if (installedHash) {
-      sources.push({ path: "plugins/installed_plugins.json", hash: installedHash });
-    }
-    let installed: unknown;
-    try {
-      installed = JSON.parse(await readFile(installedPath, "utf8"));
-    } catch {
-      installed = undefined;
-    }
-    const parsedInstalled = ClaudeInstalledPluginsFileSchema.safeParse(installed);
-    if (!parsedInstalled.success) {
-      throw new Error("enabled Claude plugins have invalid installed metadata");
-    }
-    const plugins = parsedInstalled.data.plugins;
-    for (const id of enabledIds) {
-      const records = plugins[id];
-      if (!records || records.length === 0) {
-        throw new Error(`enabled Claude plugin '${id}' has no installed content to hash`);
-      }
-      for (const [index, record] of records.entries()) {
-        const installPath = record.installPath;
-        sources.push(
-          ...(await hashBehaviorDirectory(resolve(installPath), `plugins/enabled/${id}/${index}`)),
-        );
-      }
-    }
-  }
-  return sources.length === 0 ? null : valueHash(sources);
-};
-
-export const configuredClaudeSourceHashes = async (
-  driver: unknown,
-  baseRoot: string,
-  cache: Map<string, Promise<string>> = new Map(),
-): Promise<Array<{ kind: "plugin" | "read"; path: string; hash: string }>> => {
-  const driverCli = referencedCli(driver);
-  if (!driverCli) return [];
-  const config = driverConfigRecord(driver);
-  // This also rejects path-bearing extraArgs that would evade the source
-  // hashes below. Runtime applies the same guard before driver creation.
-  boundExperimentDriverConfig(driverCli.name, config, "identity-check");
-  const isolation =
-    config.isolation && typeof config.isolation === "object"
-      ? (config.isolation as Record<string, unknown>)
-      : undefined;
-  const configured = [
-    ...(driverCli.name === "claude-code" && Array.isArray(config.pluginDirs)
-      ? config.pluginDirs.map((path) => ({ kind: "plugin" as const, path }))
-      : []),
-    ...(Array.isArray(config.addDirs)
-      ? config.addDirs.map((path) => ({ kind: "read" as const, path }))
-      : []),
-    ...(driverCli.name === "claude-code" && isolation && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead.map((path) => ({ kind: "read" as const, path }))
-      : []),
-  ].filter(
-    (entry): entry is { kind: "plugin" | "read"; path: string } =>
-      typeof entry.path === "string" && entry.path.length > 0,
-  );
-  return Promise.all(
-    configured
-      .sort((a, b) => `${a.kind}:${a.path}`.localeCompare(`${b.kind}:${b.path}`))
-      .map(async ({ kind, path }) => {
-        const absolute = resolve(baseRoot, path);
-        const key = `${kind}:${absolute}`;
-        let hash = cache.get(key);
-        if (!hash) {
-          hash = hashBehaviorDirectory(
-            absolute,
-            `driver/${kind}/${contentHash(absolute).slice(0, 12)}`,
-          ).then(valueHash);
-          cache.set(key, hash);
-        }
-        return {
-          kind,
-          path: absolute,
-          hash: await hash,
-        };
-      }),
-  );
-};
-
 export const runtimeCliIdentityHashForReferences = async (
   references: readonly RuntimeCliReference[],
 ): Promise<string> =>

package/src/orchestrator/experiment-selection.ts

@@ -13,6 +13,7 @@ import {
   ExperimentSelectionSchema,
   type OpenedExperimentRun,
 } from "./experiment-runs.js";
+import { confidenceGateIssue, type PairedEffect, pairedEffect } from "./paired-effects.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
 
@@ -103,6 +104,7 @@ const satisfiesConstraint = (
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): boolean => {
+  if (constraint.relativeTo && candidate.id === "baseline") return true;
   let profiles: (string | undefined)[];
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -110,6 +112,16 @@ const satisfiesConstraint = (
     profiles = campaign.profiles;
   }
   return profiles.every((profile) => {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return false;
+      const lower = effect.interval?.lower ?? effect.meanDelta;
+      const upper = effect.interval?.upper ?? effect.meanDelta;
+      return (
+        (constraint.min === undefined || lower >= constraint.min) &&
+        (constraint.max === undefined || upper <= constraint.max)
+      );
+    }
     const values = metricValues(
       candidate,
       constraint.metric,
@@ -135,6 +147,7 @@ const satisfiesConstraint = (
       return sampleGroups.every((samples) => {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return false;
           return (
             (constraint.min === undefined || interval.lower >= constraint.min) &&
             (constraint.max === undefined || interval.upper <= constraint.max)
@@ -156,12 +169,29 @@ const satisfiesConstraint = (
 export const average = (values: readonly number[]): number =>
   values.reduce((sum, value) => sum + value, 0) / Math.max(values.length, 1);
 
+const relativeConstraintEffect = (
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): PairedEffect => {
+  const scope = (entry: ExperimentEvaluation): boolean =>
+    entry.profileId === profile && entry.split === constraint.split;
+  return pairedEffect(
+    candidateEvaluations(experiment, "baseline").find(scope),
+    candidateEvaluations(experiment, candidate.id).find(scope),
+    constraint.metric,
+    constraint.confidence,
+  );
+};
+
 const normalizedConstraintViolation = (
   campaign: ExperimentCampaign,
   candidate: ExperimentDecisionCandidate,
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): number => {
+  if (constraint.relativeTo && candidate.id === "baseline") return 0;
   let profiles: Array<string | undefined> = campaign.profiles;
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -177,6 +207,18 @@ const normalizedConstraintViolation = (
     );
   let worst = 0;
   for (const profile of profiles) {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return Number.POSITIVE_INFINITY;
+      worst = Math.max(
+        worst,
+        violation(
+          effect.interval?.lower ?? effect.meanDelta,
+          effect.interval?.upper ?? effect.meanDelta,
+        ),
+      );
+      continue;
+    }
     if (constraint.confidence) {
       const groups = metricSampleGroups(
         candidate,
@@ -189,6 +231,7 @@ const normalizedConstraintViolation = (
       for (const samples of groups) {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return Number.POSITIVE_INFINITY;
           worst = Math.max(worst, violation(interval.lower, interval.upper));
         } catch {
           return Number.POSITIVE_INFINITY;
@@ -209,13 +252,65 @@ const normalizedConstraintViolation = (
   return worst;
 };
 
+const hasScoreEvidence = (
+  experiment: OpenedExperimentRun["manifest"],
+  split: ExperimentSplit,
+): boolean =>
+  [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
+    .flatMap((id) => candidateEvaluations(experiment, id))
+    .some(
+      (entry) =>
+        entry.split === split &&
+        (entry.metricDefinitions?.score !== undefined ||
+          entry.metrics.score !== undefined ||
+          Object.values(entry.caseSamples ?? {}).some(
+            (sample) => sample.metrics.score !== undefined,
+          )),
+    );
+
+const completeScoreEvidence = (entry: ExperimentEvaluation): boolean =>
+  entry.metrics.score !== undefined &&
+  (!entry.caseSamples ||
+    Object.keys(entry.casePassRates).every((id) => {
+      const sample = entry.caseSamples?.[id];
+      return sample?.metrics.score?.n === entry.repetitions && Boolean(sample.scoreBasis);
+    }));
+
+const comparableScoreBasis = (
+  entry: ExperimentEvaluation,
+  reference: ExperimentEvaluation | undefined,
+): boolean => {
+  if (!reference || !completeScoreEvidence(reference)) return false;
+  if (!entry.caseSamples || !reference.caseSamples)
+    return !entry.caseSamples && !reference.caseSamples;
+  const ids = new Set([
+    ...Object.keys(entry.casePassRates),
+    ...Object.keys(reference.casePassRates),
+  ]);
+  return [...ids].every((id) => {
+    const basis = entry.caseSamples?.[id]?.scoreBasis;
+    return Boolean(basis) && basis === reference.caseSamples?.[id]?.scoreBasis;
+  });
+};
+
 export const rankCandidates = <T extends ExperimentDecisionCandidate>(
   campaign: ExperimentCampaign,
   candidates: readonly T[],
   experiment: OpenedExperimentRun["manifest"],
   constraintSplits: readonly ExperimentSplit[] = ["train", "validation", "test"],
-): T[] =>
-  [...candidates].sort((a, b) => {
+): T[] => {
+  const validationEvaluations = (id: string): ExperimentEvaluation[] =>
+    candidateEvaluations(experiment, id).filter((entry) => entry.split === "validation");
+  const referenceId = ["baseline", ...candidates.map((candidate) => candidate.id).sort()].find(
+    (id) => {
+      const entries = validationEvaluations(id);
+      return campaign.profiles.every((profile) =>
+        entries.some((entry) => entry.profileId === profile && completeScoreEvidence(entry)),
+      );
+    },
+  );
+  const reference = referenceId ? validationEvaluations(referenceId) : [];
+  return [...candidates].sort((a, b) => {
     const activeConstraints = campaign.objectives.constraints.filter((constraint) =>
       constraintSplits.includes(constraint.split),
     );
@@ -267,20 +362,40 @@ export const rankCandidates = <T extends ExperimentDecisionCandidate>(
       }
     }
     const quality = (candidate: ExperimentDecisionCandidate): number => {
-      const values = metricValues(
-        candidate,
-        "quality.passRate",
-        undefined,
-        "validation",
-        experiment,
-      );
-      return values.length > 0 ? Math.min(...values) : -1;
+      if (!hasScoreEvidence(experiment, "validation")) {
+        const values = metricValues(
+          candidate,
+          "quality.passRate",
+          undefined,
+          "validation",
+          experiment,
+        );
+        return values.length > 0 ? Math.min(...values) : -1;
+      }
+      const evaluations = candidateEvaluations(experiment, candidate.id);
+      const scores = campaign.profiles.map((profile) => {
+        const entry = evaluations.find(
+          (value) => value.profileId === profile && value.split === "validation",
+        );
+        if (
+          !entry ||
+          !completeScoreEvidence(entry) ||
+          !comparableScoreBasis(
+            entry,
+            reference.find((value) => value.profileId === profile),
+          )
+        )
+          return -1;
+        return entry.metrics.score ?? -1;
+      });
+      return scores.length > 0 ? Math.min(...scores) : -1;
     };
     const qualityDelta = quality(b) - quality(a);
     if (qualityDelta !== 0) return qualityDelta;
     if (a.sourceBytes !== b.sourceBytes) return a.sourceBytes - b.sourceBytes;
     return a.id.localeCompare(b.id);
   });
+};
 
 const candidateCaseVector = (
   candidate: Candidate,
@@ -295,10 +410,15 @@ const candidateCaseVector = (
     evaluations
       .filter((entry) => entry.split === split)
       .flatMap((entry) =>
-        Object.entries(entry.casePassRates).map(([caseId, passRate]) => [
-          `${entry.profileId}:${caseId}`,
-          passRate,
-        ]),
+        Object.entries(entry.casePassRates).flatMap(([caseId, passRate]) => {
+          if (!hasScoreEvidence(experiment, split))
+            return [[`${entry.profileId}:${caseId}`, passRate]];
+          const sample = entry.caseSamples?.[caseId];
+          const score = sample?.metrics.score;
+          return score?.n === entry.repetitions && sample?.scoreBasis
+            ? [[JSON.stringify([entry.profileId, caseId, sample.scoreBasis]), score.mean]]
+            : [];
+        }),
       ),
   );
 };
@@ -308,8 +428,9 @@ const dominates = (a: Record<string, number>, b: Record<string, number>): boolea
   if (keys.size === 0) return false;
   let strictlyBetter = false;
   for (const key of keys) {
-    const av = a[key] ?? 0;
-    const bv = b[key] ?? 0;
+    const av = a[key];
+    const bv = b[key];
+    if (av === undefined || bv === undefined) return false;
     if (av < bv) return false;
     if (av > bv) strictlyBetter = true;
   }
@@ -361,8 +482,13 @@ const frontierCoverage = (
   const candidateVector = candidateCaseVector(candidate, experiment, "train");
   const keys = new Set(vectors.flatMap((vector) => Object.keys(vector)));
   return [...keys].filter((key) => {
-    const best = Math.max(...vectors.map((vector) => vector[key] ?? 0));
-    return (candidateVector[key] ?? 0) === best;
+    const values = vectors.flatMap((vector) => (vector[key] === undefined ? [] : [vector[key]]));
+    const best = Math.max(...values);
+    return (
+      (frontier.length === 1 || values.length > 1) &&
+      candidateVector[key] !== undefined &&
+      candidateVector[key] === best
+    );
   }).length;
 };
 
@@ -379,6 +505,36 @@ export const searchFrontier = (
   });
 };
 
+const relativeConstraintReport = (
+  campaign: ExperimentCampaign,
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): ExperimentSelection["constraints"][number] => {
+  const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+  const reference = candidate.id === "baseline";
+  let values = effect.meanDelta === null ? [] : [effect.meanDelta];
+  if (reference) values = [0];
+  const note = reference ? "baseline reference; relative gate does not apply" : effect.note;
+  return {
+    candidateId: candidate.id,
+    metric: constraint.metric,
+    relativeTo: "baseline",
+    ...(profile ? { profile } : {}),
+    split: constraint.split,
+    values,
+    passed: satisfiesConstraint(
+      campaign,
+      candidate,
+      { ...constraint, ...(profile ? { profile } : {}) },
+      experiment,
+    ),
+    ...(!reference && effect.interval ? { interval: effect.interval } : {}),
+    ...(note ? { note } : {}),
+  };
+};
+
 export const deriveExperimentSelection = (
   campaign: ExperimentCampaign,
   candidates: readonly ExperimentDecisionCandidate[],
@@ -405,6 +561,9 @@ export const deriveExperimentSelection = (
           ? [constraint.profile]
           : campaign.profiles;
       return profiles.map((profile) => {
+        if (constraint.relativeTo) {
+          return relativeConstraintReport(campaign, candidate, constraint, profile, experiment);
+        }
         const values = metricValues(
           candidate,
           constraint.metric,
@@ -426,6 +585,7 @@ export const deriveExperimentSelection = (
           if (samples) {
             try {
               interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+              note = confidenceGateIssue(interval);
             } catch (error) {
               note = error instanceof Error ? error.message : String(error);
             }