@prismatic-io/lux 0.0.2-preview.14 → 0.0.2-preview.16

package/src/orchestrator/experiment-identity.ts

@@ -5,20 +5,17 @@ import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
 import { authoredDependencyGraphHash } from "./authored-dependencies.js";
 import { assertValidationArgumentsBound } from "./candidates.js";
 import { selectCases } from "./discover.js";
+import { experimentDriverIdentity, referencedDriverRuntime } from "./driver-identity.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { caseRubricCliReferences, selectorsBySplit } from "./experiment-corpus.js";
 import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
   cliIdentity,
-  configuredClaudeSourceHashes,
   LUX_VERSION,
   type RuntimeCliReference,
   RuntimeCliReferenceSchema,
   referencedAnswererCli,
-  referencedCli,
   runtimeCliIdentityHashForReferences,
 } from "./experiment-runtime-identity.js";
 import { loadEvalCase } from "./loader.js";
@@ -33,15 +30,6 @@ import { validatePluginConfig } from "./validate-plugin-config.js";
 export const resolvedProfileSnapshot = async (
   execution: CampaignExecutionOptions,
 ): Promise<Record<string, unknown>> => {
-  const behaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
-  const behaviorHash = (root: string): Promise<string | null> => {
-    const existing = behaviorHashes.get(root);
-    if (existing) return existing;
-    const created = claudeUserBehaviorHash(root);
-    behaviorHashes.set(root, created);
-    return created;
-  };
   return Object.fromEntries(
     await Promise.all(
       execution.loaded.campaign.profiles.map(async (profileId) => {
@@ -53,20 +41,12 @@ export const resolvedProfileSnapshot = async (
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.driver ?? execution.config.defaultDriver)
             : undefined;
-        const driverCli = referencedCli(driver);
         const harness = profile.harness ?? execution.config.harness;
         const answerer =
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.answerer ?? execution.config.defaultAnswerer)
             : undefined;
         const answererCli = referencedAnswererCli(answerer);
-        const driverUserBehaviorHash =
-          driverCli?.name === "claude-code" ? await behaviorHash(claudeConfigRoot(driver)) : null;
-        const driverConfiguredSources = await configuredClaudeSourceHashes(
-          driver,
-          execution.config.rootDir,
-          configuredSourceHashes,
-        );
         let harnessCommand = harness?.command;
         if (!harnessCommand) {
           harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
@@ -80,17 +60,15 @@ export const resolvedProfileSnapshot = async (
           profileId,
           {
             id: profileId,
+            environment: profile.environment ?? execution.config.environment ?? null,
+            configuredConcurrency: execution.config.concurrency,
+            evaluationConcurrency: 1,
             driver: driver ?? null,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            ...(driverCli?.name === "claude-code"
-              ? {
-                  driverClaudeConfigRoot: claudeConfigRoot(driver),
-                  driverUserBehaviorHash,
-                  driverConfiguredSources,
-                }
-              : {}),
+            driverIdentity: await experimentDriverIdentity(
+              driver,
+              execution.registry,
+              execution.config.rootDir,
+            ),
             answerer,
             answererCliIdentity: answererCli
               ? await cliIdentity(answererCli.command, execution.config.rootDir)
@@ -130,7 +108,11 @@ const trustedExecutableEntrypoints = async (
   }
   for (const profileId of execution.loaded.campaign.profiles) {
     const profile = profileOrThrow(execution.config, profileId);
-    add(referencedCli(profile.driver ?? execution.config.defaultDriver)?.command);
+    for (const command of referencedDriverRuntime(
+      profile.driver ?? execution.config.defaultDriver,
+      execution.registry,
+    ).commands)
+      add(command);
     add(referencedAnswererCli(profile.answerer ?? execution.config.defaultAnswerer)?.command);
     add((profile.harness ?? execution.config.harness)?.command);
   }
@@ -147,7 +129,15 @@ const trustedExecutableEntrypoints = async (
         await loadEvalCase(evalCase.path),
         execution.loaded.subjectRoot,
       );
-      add(referencedCli(authored.driver)?.command);
+      for (const profileId of execution.loaded.campaign.profiles) {
+        const profile = profileOrThrow(execution.config, profileId);
+        const driver = resolveDriverReference(
+          authored,
+          profile.driver ?? execution.config.defaultDriver,
+        );
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command);
+      }
       add(referencedAnswererCli(authored.answerer)?.command);
       for (const reference of caseRubricCliReferences(execution, authored)) {
         add(reference.command);
@@ -216,7 +206,8 @@ export const runtimeCliReferences = async (
           profile.answerer ?? execution.config.defaultAnswerer,
           harness,
         );
-        add(referencedCli(driver)?.command, execution.config.rootDir);
+        for (const command of referencedDriverRuntime(driver, execution.registry).commands)
+          add(command, execution.config.rootDir);
         add(referencedAnswererCli(answerer)?.command, execution.config.rootDir);
       }
       for (const reference of caseRubricCliReferences(execution, authored)) {
@@ -286,7 +277,6 @@ const validateAgentCaseProfileDrivers = async (
       );
       const name = typeof reference === "string" ? reference : reference.name;
       const rawConfig = typeof reference === "object" ? reference.config : undefined;
-      boundExperimentDriverConfig(name, rawConfig, "identity-check");
       const plugin = execution.registry.getDriver(name);
       if (!plugin) {
         throw new Error(
@@ -298,7 +288,7 @@ const validateAgentCaseProfileDrivers = async (
         rawConfig,
         `case '${evalCase.id}' profile '${profileId}' driver '${name}'`,
       );
-      boundExperimentDriverConfig(name, validated, "identity-check");
+      boundExperimentDriverConfig(plugin, validated, "identity-check");
     }
   }
 };
@@ -334,6 +324,7 @@ export const evaluationIdentity = async (
     luxVersion: LUX_VERSION,
     evaluatorImplementationHash: await evaluatorImplementationHash(),
     runtime: runtimeIdentity(),
+    concurrency: execution.config.concurrency,
     corpusHash: corpus.hash,
     profiles,
     configSourceHash,

package/src/orchestrator/experiment-report.ts

@@ -2,6 +2,7 @@ import { basename } from "node:path";
 import { z } from "zod";
 import { valueHash } from "../core/index.js";
 import { deriveExperimentSelection, rankCandidates } from "./experiment.js";
+import { ExperimentEffectsSchema, experimentEffects } from "./experiment-effects.js";
 import {
   type ExperimentRun,
   ExperimentRunSchema,
@@ -24,6 +25,7 @@ export const ExperimentDecisionReportSchema = z
     status: z.enum(["running", "passed", "failed", "aborted"]),
     outcome: z.enum(["promote", "retain", "reject", "error", "incomplete"]),
     error: z.string().nullable(),
+    effects: z.exactOptional(ExperimentEffectsSchema),
     candidateAndAccountingVerified: z.boolean(),
     championCandidateId: z.string().nullable(),
     ranking: z.array(z.string()),
@@ -39,6 +41,7 @@ export const ExperimentDecisionReportSchema = z
       z
         .object({
           metric: z.string(),
+          relativeTo: z.exactOptional(z.literal("baseline")),
           profile: z.string().nullable(),
           split: ExperimentReportSplitSchema,
           passed: z.boolean(),
@@ -596,6 +599,7 @@ const projectExperimentDecisionReport = (
     evaluationKind: experiment.evaluationKind,
     status: experiment.status,
     error: experiment.errorMessage ?? null,
+    effects: experimentEffects(experiment),
     candidateAndAccountingVerified,
     outcome: reportOutcome(experiment, championId),
     championCandidateId: championId,
@@ -610,6 +614,7 @@ const projectExperimentDecisionReport = (
       .filter((constraint) => constraint.candidateId === targetId)
       .map((constraint) => ({
         metric: constraint.metric,
+        ...(constraint.relativeTo ? { relativeTo: constraint.relativeTo } : {}),
         profile: constraint.profile ?? null,
         split: constraint.split,
         passed: constraint.passed,
@@ -733,7 +738,7 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       "| --- | --- | --- | --- | ---: | --- | --- |",
       ...report.gates.map(
         (gate) =>
-          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
+          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)}${gate.relativeTo ? " Δ baseline" : ""} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
       ),
     );
   }
@@ -763,6 +768,21 @@ export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionRepor
       ),
     );
   }
+  if (report.effects?.length) {
+    lines.push(
+      "",
+      "## Paired case effects",
+      "",
+      "Differences are candidate minus baseline; repetitions are averaged within cases and authored clusters receive equal weight.",
+      "",
+      "| Candidate | Profile | Split | Metric | Mean difference | Cases | Clusters | Confidence | Note |",
+      "| --- | --- | --- | --- | ---: | ---: | ---: | --- | --- |",
+      ...report.effects.map(
+        (effect) =>
+          `| ${markdown(effect.candidateId)} | ${markdown(effect.profileId)} | ${effect.split} | ${markdown(effect.metric)} | ${effect.meanDelta === null ? "—" : signed(effect.meanDelta)} | ${effect.caseCount ?? effect.caseIds.length} | ${effect.n} | ${markdown(intervalText(effect.interval ?? undefined) ?? "—")} | ${markdown(effect.note ?? "—")} |`,
+      ),
+    );
+  }
   lines.push(
     "",
     "## Source and spend",

package/src/orchestrator/experiment-runs.ts

@@ -86,6 +86,7 @@ export const ExperimentSelectionSchema = z.object({
     z.object({
       candidateId: CandidateIdSchema,
       metric: z.string(),
+      relativeTo: z.exactOptional(z.literal("baseline")),
       profile: z.exactOptional(z.string()),
       split: z.enum(["train", "validation", "test"]),
       values: z.array(z.number()),
@@ -738,6 +739,7 @@ export const resumeExperimentRun = async (path: string): Promise<OpenedExperimen
 
 export const listExperimentRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<ExperimentRun & { experimentDir: string }>> => {
   const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
   if (!existsSync(root)) return [];
@@ -745,10 +747,15 @@ export const listExperimentRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const runs = await Promise.all(
-    directories.map(async (experimentDir) => ({
-      experimentDir,
-      experiment: await loadExperimentRun(experimentDir),
-    })),
+    directories.map(async (experimentDir) => {
+      try {
+        return { experimentDir, experiment: await loadExperimentRun(experimentDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(experimentDir, error);
+        return { experimentDir, experiment: null };
+      }
+    }),
   );
   return runs
     .flatMap(({ experimentDir, experiment }) =>

package/src/orchestrator/experiment-runtime-identity.ts

@@ -1,61 +1,14 @@
 import { createHash } from "node:crypto";
-import { access, constants, lstat, open, readFile, realpath } from "node:fs/promises";
-import { homedir } from "node:os";
+import { access, constants, lstat, open, realpath } from "node:fs/promises";
 import { delimiter, isAbsolute, join, resolve } from "node:path";
 import { z } from "zod";
-import { contentHash, discoverCliVersion, valueHash } from "../core/index.js";
-import { walkAuthoredFiles } from "./fs-read.js";
-import { boundExperimentDriverConfig } from "./orchestrator.js";
+import { discoverCliVersion, valueHash } from "../core/index.js";
 
 export { LUX_VERSION } from "./package-version.js";
 
-const ClaudeSettingsFileSchema = z.looseObject({
-  enabledPlugins: z.record(z.string(), z.boolean()).optional(),
-});
-const ClaudeInstalledPluginRecordSchema = z.looseObject({
-  installPath: z.string(),
-});
-const ClaudeInstalledPluginsFileSchema = z.looseObject({
-  plugins: z.record(z.string(), z.array(ClaudeInstalledPluginRecordSchema)),
-});
-
-export const referencedCli = (
-  reference: unknown,
-): { name: string; command: string } | undefined => {
-  let name: string | undefined;
-  if (typeof reference === "string") {
-    name = reference;
-  } else if (
-    typeof reference === "object" &&
-    reference !== null &&
-    typeof (reference as { name?: unknown }).name === "string"
-  ) {
-    name = (reference as { name: string }).name;
-  }
-  const config =
-    typeof reference === "object" && reference !== null
-      ? (reference as { config?: unknown }).config
-      : undefined;
-  if (name === "subprocess") {
-    if (
-      typeof config !== "object" ||
-      config === null ||
-      typeof (config as { command?: unknown }).command !== "string"
-    ) {
-      return undefined;
-    }
-    return { name, command: (config as { command: string }).command };
-  }
-  if (name !== "claude-code" && name !== "codex") return undefined;
-  let command = name === "codex" ? "codex" : "claude";
-  if (
-    typeof config === "object" &&
-    config !== null &&
-    typeof (config as { command?: unknown }).command === "string"
-  ) {
-    command = (config as { command: string }).command;
-  }
-  return { name, command };
+const driverConfigRecord = (reference: unknown): Record<string, unknown> => {
+  const config = (reference as { config?: unknown } | null)?.config;
+  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
 };
 
 export const referencedAnswererCli = (
@@ -261,197 +214,6 @@ export const cliIdentity = async (command: string, baseRoot: string): Promise<Cl
   });
 };
 
-const optionalRegularFileHash = async (path: string): Promise<string | null> => {
-  try {
-    const info = await lstat(path);
-    if (info.isSymbolicLink() || !info.isFile()) {
-      throw new Error(`behavior-affecting settings must be a regular file: ${path}`);
-    }
-    return contentHash(await readFile(path));
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
-    throw error;
-  }
-};
-
-const driverConfigRecord = (driver: unknown): Record<string, unknown> => {
-  if (!driver || typeof driver !== "object") return {};
-  const config = (driver as { config?: unknown }).config;
-  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
-};
-
-export const claudeConfigRoot = (driver: unknown): string => {
-  const config = driverConfigRecord(driver);
-  const env =
-    config.env && typeof config.env === "object"
-      ? (config.env as Record<string, unknown>)
-      : undefined;
-  const configured = env?.CLAUDE_CONFIG_DIR;
-  const root =
-    typeof configured === "string"
-      ? configured
-      : (process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"));
-  if (!isAbsolute(root)) {
-    throw new Error("CLAUDE_CONFIG_DIR must be absolute for reproducible experiment identity");
-  }
-  return resolve(root);
-};
-
-const hashBehaviorDirectory = async (
-  root: string,
-  label: string,
-): Promise<Array<{ path: string; hash: string }>> => {
-  try {
-    const info = await lstat(root);
-    if (info.isSymbolicLink() || !info.isDirectory()) {
-      throw new Error(`behavior-affecting Claude source must be a real directory: ${root}`);
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
-    throw error;
-  }
-  const paths = await walkAuthoredFiles(root);
-  if (paths.length > 5_000) {
-    throw new Error(`behavior-affecting Claude source '${label}' exceeds 5000 files`);
-  }
-  let bytes = 0;
-  const hashes: Array<{ path: string; hash: string }> = [];
-  for (const path of paths) {
-    const content = await readFile(join(root, path));
-    bytes += content.length;
-    if (bytes > 256 * 1024 * 1024) {
-      throw new Error(`behavior-affecting Claude source '${label}' exceeds 256 MiB`);
-    }
-    hashes.push({ path: `${label}/${path}`, hash: contentHash(content) });
-  }
-  return hashes;
-};
-
-/**
- * Hash every intentionally loaded user behavior source without persisting its
- * contents. This covers user instructions/rules/output styles and the exact
- * installed contents of plugins enabled by settings.json.
- */
-export const claudeUserBehaviorHash = async (configRoot: string): Promise<string | null> => {
-  const settingsPath = join(configRoot, "settings.json");
-  const settingsHash = await optionalRegularFileHash(settingsPath);
-  const sources: Array<{ path: string; hash: string }> = [];
-  if (settingsHash) sources.push({ path: "settings.json", hash: settingsHash });
-  const claudeMdHash = await optionalRegularFileHash(join(configRoot, "CLAUDE.md"));
-  if (claudeMdHash) sources.push({ path: "CLAUDE.md", hash: claudeMdHash });
-  for (const directory of ["rules", "output-styles", "agents", "commands", "skills"]) {
-    sources.push(...(await hashBehaviorDirectory(join(configRoot, directory), directory)));
-  }
-
-  let settings: unknown;
-  try {
-    const contents = await readFile(settingsPath, "utf8");
-    try {
-      settings = JSON.parse(contents);
-    } catch {
-      throw new Error("Claude settings.json contains invalid JSON");
-    }
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      settings = undefined;
-    } else {
-      throw error;
-    }
-  }
-  const parsedSettings = ClaudeSettingsFileSchema.safeParse(settings);
-  if (settings !== undefined && !parsedSettings.success) {
-    throw new Error("Claude settings.json has an invalid enabledPlugins map");
-  }
-  const enabledPlugins = parsedSettings.success ? parsedSettings.data.enabledPlugins : undefined;
-  const enabledIds = Object.entries(enabledPlugins ?? {})
-    .filter(([, value]) => value === true)
-    .map(([id]) => id)
-    .sort();
-  if (enabledIds.length > 0) {
-    const installedPath = join(configRoot, "plugins", "installed_plugins.json");
-    const installedHash = await optionalRegularFileHash(installedPath);
-    if (installedHash) {
-      sources.push({ path: "plugins/installed_plugins.json", hash: installedHash });
-    }
-    let installed: unknown;
-    try {
-      installed = JSON.parse(await readFile(installedPath, "utf8"));
-    } catch {
-      installed = undefined;
-    }
-    const parsedInstalled = ClaudeInstalledPluginsFileSchema.safeParse(installed);
-    if (!parsedInstalled.success) {
-      throw new Error("enabled Claude plugins have invalid installed metadata");
-    }
-    const plugins = parsedInstalled.data.plugins;
-    for (const id of enabledIds) {
-      const records = plugins[id];
-      if (!records || records.length === 0) {
-        throw new Error(`enabled Claude plugin '${id}' has no installed content to hash`);
-      }
-      for (const [index, record] of records.entries()) {
-        const installPath = record.installPath;
-        sources.push(
-          ...(await hashBehaviorDirectory(resolve(installPath), `plugins/enabled/${id}/${index}`)),
-        );
-      }
-    }
-  }
-  return sources.length === 0 ? null : valueHash(sources);
-};
-
-export const configuredClaudeSourceHashes = async (
-  driver: unknown,
-  baseRoot: string,
-  cache: Map<string, Promise<string>> = new Map(),
-): Promise<Array<{ kind: "plugin" | "read"; path: string; hash: string }>> => {
-  const driverCli = referencedCli(driver);
-  if (!driverCli) return [];
-  const config = driverConfigRecord(driver);
-  // This also rejects path-bearing extraArgs that would evade the source
-  // hashes below. Runtime applies the same guard before driver creation.
-  boundExperimentDriverConfig(driverCli.name, config, "identity-check");
-  const isolation =
-    config.isolation && typeof config.isolation === "object"
-      ? (config.isolation as Record<string, unknown>)
-      : undefined;
-  const configured = [
-    ...(driverCli.name === "claude-code" && Array.isArray(config.pluginDirs)
-      ? config.pluginDirs.map((path) => ({ kind: "plugin" as const, path }))
-      : []),
-    ...(Array.isArray(config.addDirs)
-      ? config.addDirs.map((path) => ({ kind: "read" as const, path }))
-      : []),
-    ...(driverCli.name === "claude-code" && isolation && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead.map((path) => ({ kind: "read" as const, path }))
-      : []),
-  ].filter(
-    (entry): entry is { kind: "plugin" | "read"; path: string } =>
-      typeof entry.path === "string" && entry.path.length > 0,
-  );
-  return Promise.all(
-    configured
-      .sort((a, b) => `${a.kind}:${a.path}`.localeCompare(`${b.kind}:${b.path}`))
-      .map(async ({ kind, path }) => {
-        const absolute = resolve(baseRoot, path);
-        const key = `${kind}:${absolute}`;
-        let hash = cache.get(key);
-        if (!hash) {
-          hash = hashBehaviorDirectory(
-            absolute,
-            `driver/${kind}/${contentHash(absolute).slice(0, 12)}`,
-          ).then(valueHash);
-          cache.set(key, hash);
-        }
-        return {
-          kind,
-          path: absolute,
-          hash: await hash,
-        };
-      }),
-  );
-};
-
 export const runtimeCliIdentityHashForReferences = async (
   references: readonly RuntimeCliReference[],
 ): Promise<string> =>

package/src/orchestrator/experiment-selection.ts

@@ -13,6 +13,7 @@ import {
   ExperimentSelectionSchema,
   type OpenedExperimentRun,
 } from "./experiment-runs.js";
+import { confidenceGateIssue, type PairedEffect, pairedEffect } from "./paired-effects.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
 
@@ -103,6 +104,7 @@ const satisfiesConstraint = (
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): boolean => {
+  if (constraint.relativeTo && candidate.id === "baseline") return true;
   let profiles: (string | undefined)[];
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -110,6 +112,16 @@ const satisfiesConstraint = (
     profiles = campaign.profiles;
   }
   return profiles.every((profile) => {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return false;
+      const lower = effect.interval?.lower ?? effect.meanDelta;
+      const upper = effect.interval?.upper ?? effect.meanDelta;
+      return (
+        (constraint.min === undefined || lower >= constraint.min) &&
+        (constraint.max === undefined || upper <= constraint.max)
+      );
+    }
     const values = metricValues(
       candidate,
       constraint.metric,
@@ -135,6 +147,7 @@ const satisfiesConstraint = (
       return sampleGroups.every((samples) => {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return false;
           return (
             (constraint.min === undefined || interval.lower >= constraint.min) &&
             (constraint.max === undefined || interval.upper <= constraint.max)
@@ -156,12 +169,29 @@ const satisfiesConstraint = (
 export const average = (values: readonly number[]): number =>
   values.reduce((sum, value) => sum + value, 0) / Math.max(values.length, 1);
 
+const relativeConstraintEffect = (
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): PairedEffect => {
+  const scope = (entry: ExperimentEvaluation): boolean =>
+    entry.profileId === profile && entry.split === constraint.split;
+  return pairedEffect(
+    candidateEvaluations(experiment, "baseline").find(scope),
+    candidateEvaluations(experiment, candidate.id).find(scope),
+    constraint.metric,
+    constraint.confidence,
+  );
+};
+
 const normalizedConstraintViolation = (
   campaign: ExperimentCampaign,
   candidate: ExperimentDecisionCandidate,
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): number => {
+  if (constraint.relativeTo && candidate.id === "baseline") return 0;
   let profiles: Array<string | undefined> = campaign.profiles;
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -177,6 +207,18 @@ const normalizedConstraintViolation = (
     );
   let worst = 0;
   for (const profile of profiles) {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return Number.POSITIVE_INFINITY;
+      worst = Math.max(
+        worst,
+        violation(
+          effect.interval?.lower ?? effect.meanDelta,
+          effect.interval?.upper ?? effect.meanDelta,
+        ),
+      );
+      continue;
+    }
     if (constraint.confidence) {
       const groups = metricSampleGroups(
         candidate,
@@ -189,6 +231,7 @@ const normalizedConstraintViolation = (
       for (const samples of groups) {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return Number.POSITIVE_INFINITY;
           worst = Math.max(worst, violation(interval.lower, interval.upper));
         } catch {
           return Number.POSITIVE_INFINITY;
@@ -379,6 +422,36 @@ export const searchFrontier = (
   });
 };
 
+const relativeConstraintReport = (
+  campaign: ExperimentCampaign,
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): ExperimentSelection["constraints"][number] => {
+  const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+  const reference = candidate.id === "baseline";
+  let values = effect.meanDelta === null ? [] : [effect.meanDelta];
+  if (reference) values = [0];
+  const note = reference ? "baseline reference; relative gate does not apply" : effect.note;
+  return {
+    candidateId: candidate.id,
+    metric: constraint.metric,
+    relativeTo: "baseline",
+    ...(profile ? { profile } : {}),
+    split: constraint.split,
+    values,
+    passed: satisfiesConstraint(
+      campaign,
+      candidate,
+      { ...constraint, ...(profile ? { profile } : {}) },
+      experiment,
+    ),
+    ...(!reference && effect.interval ? { interval: effect.interval } : {}),
+    ...(note ? { note } : {}),
+  };
+};
+
 export const deriveExperimentSelection = (
   campaign: ExperimentCampaign,
   candidates: readonly ExperimentDecisionCandidate[],
@@ -405,6 +478,9 @@ export const deriveExperimentSelection = (
           ? [constraint.profile]
           : campaign.profiles;
       return profiles.map((profile) => {
+        if (constraint.relativeTo) {
+          return relativeConstraintReport(campaign, candidate, constraint, profile, experiment);
+        }
         const values = metricValues(
           candidate,
           constraint.metric,
@@ -426,6 +502,7 @@ export const deriveExperimentSelection = (
           if (samples) {
             try {
               interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+              note = confidenceGateIssue(interval);
             } catch (error) {
               note = error instanceof Error ? error.message : String(error);
             }