@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/experiment-selection.ts

@@ -13,6 +13,7 @@ import {
   ExperimentSelectionSchema,
   type OpenedExperimentRun,
 } from "./experiment-runs.js";
+import { confidenceGateIssue, type PairedEffect, pairedEffect } from "./paired-effects.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
 
@@ -103,6 +104,7 @@ const satisfiesConstraint = (
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): boolean => {
+  if (constraint.relativeTo && candidate.id === "baseline") return true;
   let profiles: (string | undefined)[];
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -110,6 +112,16 @@ const satisfiesConstraint = (
     profiles = campaign.profiles;
   }
   return profiles.every((profile) => {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return false;
+      const lower = effect.interval?.lower ?? effect.meanDelta;
+      const upper = effect.interval?.upper ?? effect.meanDelta;
+      return (
+        (constraint.min === undefined || lower >= constraint.min) &&
+        (constraint.max === undefined || upper <= constraint.max)
+      );
+    }
     const values = metricValues(
       candidate,
       constraint.metric,
@@ -135,6 +147,7 @@ const satisfiesConstraint = (
       return sampleGroups.every((samples) => {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return false;
           return (
             (constraint.min === undefined || interval.lower >= constraint.min) &&
             (constraint.max === undefined || interval.upper <= constraint.max)
@@ -156,12 +169,29 @@ const satisfiesConstraint = (
 export const average = (values: readonly number[]): number =>
   values.reduce((sum, value) => sum + value, 0) / Math.max(values.length, 1);
 
+const relativeConstraintEffect = (
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): PairedEffect => {
+  const scope = (entry: ExperimentEvaluation): boolean =>
+    entry.profileId === profile && entry.split === constraint.split;
+  return pairedEffect(
+    candidateEvaluations(experiment, "baseline").find(scope),
+    candidateEvaluations(experiment, candidate.id).find(scope),
+    constraint.metric,
+    constraint.confidence,
+  );
+};
+
 const normalizedConstraintViolation = (
   campaign: ExperimentCampaign,
   candidate: ExperimentDecisionCandidate,
   constraint: ExperimentConstraint,
   experiment: OpenedExperimentRun["manifest"],
 ): number => {
+  if (constraint.relativeTo && candidate.id === "baseline") return 0;
   let profiles: Array<string | undefined> = campaign.profiles;
   if (constraint.metric === "subject.bytes" || constraint.profile) {
     profiles = [constraint.profile];
@@ -177,6 +207,18 @@ const normalizedConstraintViolation = (
     );
   let worst = 0;
   for (const profile of profiles) {
+    if (constraint.relativeTo) {
+      const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+      if (effect.note || effect.meanDelta === null) return Number.POSITIVE_INFINITY;
+      worst = Math.max(
+        worst,
+        violation(
+          effect.interval?.lower ?? effect.meanDelta,
+          effect.interval?.upper ?? effect.meanDelta,
+        ),
+      );
+      continue;
+    }
     if (constraint.confidence) {
       const groups = metricSampleGroups(
         candidate,
@@ -189,6 +231,7 @@ const normalizedConstraintViolation = (
       for (const samples of groups) {
         try {
           const interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+          if (confidenceGateIssue(interval)) return Number.POSITIVE_INFINITY;
           worst = Math.max(worst, violation(interval.lower, interval.upper));
         } catch {
           return Number.POSITIVE_INFINITY;
@@ -209,13 +252,65 @@ const normalizedConstraintViolation = (
   return worst;
 };
 
+const hasScoreEvidence = (
+  experiment: OpenedExperimentRun["manifest"],
+  split: ExperimentSplit,
+): boolean =>
+  [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
+    .flatMap((id) => candidateEvaluations(experiment, id))
+    .some(
+      (entry) =>
+        entry.split === split &&
+        (entry.metricDefinitions?.score !== undefined ||
+          entry.metrics.score !== undefined ||
+          Object.values(entry.caseSamples ?? {}).some(
+            (sample) => sample.metrics.score !== undefined,
+          )),
+    );
+
+const completeScoreEvidence = (entry: ExperimentEvaluation): boolean =>
+  entry.metrics.score !== undefined &&
+  (!entry.caseSamples ||
+    Object.keys(entry.casePassRates).every((id) => {
+      const sample = entry.caseSamples?.[id];
+      return sample?.metrics.score?.n === entry.repetitions && Boolean(sample.scoreBasis);
+    }));
+
+const comparableScoreBasis = (
+  entry: ExperimentEvaluation,
+  reference: ExperimentEvaluation | undefined,
+): boolean => {
+  if (!reference || !completeScoreEvidence(reference)) return false;
+  if (!entry.caseSamples || !reference.caseSamples)
+    return !entry.caseSamples && !reference.caseSamples;
+  const ids = new Set([
+    ...Object.keys(entry.casePassRates),
+    ...Object.keys(reference.casePassRates),
+  ]);
+  return [...ids].every((id) => {
+    const basis = entry.caseSamples?.[id]?.scoreBasis;
+    return Boolean(basis) && basis === reference.caseSamples?.[id]?.scoreBasis;
+  });
+};
+
 export const rankCandidates = <T extends ExperimentDecisionCandidate>(
   campaign: ExperimentCampaign,
   candidates: readonly T[],
   experiment: OpenedExperimentRun["manifest"],
   constraintSplits: readonly ExperimentSplit[] = ["train", "validation", "test"],
-): T[] =>
-  [...candidates].sort((a, b) => {
+): T[] => {
+  const validationEvaluations = (id: string): ExperimentEvaluation[] =>
+    candidateEvaluations(experiment, id).filter((entry) => entry.split === "validation");
+  const referenceId = ["baseline", ...candidates.map((candidate) => candidate.id).sort()].find(
+    (id) => {
+      const entries = validationEvaluations(id);
+      return campaign.profiles.every((profile) =>
+        entries.some((entry) => entry.profileId === profile && completeScoreEvidence(entry)),
+      );
+    },
+  );
+  const reference = referenceId ? validationEvaluations(referenceId) : [];
+  return [...candidates].sort((a, b) => {
     const activeConstraints = campaign.objectives.constraints.filter((constraint) =>
       constraintSplits.includes(constraint.split),
     );
@@ -267,20 +362,40 @@ export const rankCandidates = <T extends ExperimentDecisionCandidate>(
       }
     }
     const quality = (candidate: ExperimentDecisionCandidate): number => {
-      const values = metricValues(
-        candidate,
-        "quality.passRate",
-        undefined,
-        "validation",
-        experiment,
-      );
-      return values.length > 0 ? Math.min(...values) : -1;
+      if (!hasScoreEvidence(experiment, "validation")) {
+        const values = metricValues(
+          candidate,
+          "quality.passRate",
+          undefined,
+          "validation",
+          experiment,
+        );
+        return values.length > 0 ? Math.min(...values) : -1;
+      }
+      const evaluations = candidateEvaluations(experiment, candidate.id);
+      const scores = campaign.profiles.map((profile) => {
+        const entry = evaluations.find(
+          (value) => value.profileId === profile && value.split === "validation",
+        );
+        if (
+          !entry ||
+          !completeScoreEvidence(entry) ||
+          !comparableScoreBasis(
+            entry,
+            reference.find((value) => value.profileId === profile),
+          )
+        )
+          return -1;
+        return entry.metrics.score ?? -1;
+      });
+      return scores.length > 0 ? Math.min(...scores) : -1;
     };
     const qualityDelta = quality(b) - quality(a);
     if (qualityDelta !== 0) return qualityDelta;
     if (a.sourceBytes !== b.sourceBytes) return a.sourceBytes - b.sourceBytes;
     return a.id.localeCompare(b.id);
   });
+};
 
 const candidateCaseVector = (
   candidate: Candidate,
@@ -295,10 +410,15 @@ const candidateCaseVector = (
     evaluations
       .filter((entry) => entry.split === split)
       .flatMap((entry) =>
-        Object.entries(entry.casePassRates).map(([caseId, passRate]) => [
-          `${entry.profileId}:${caseId}`,
-          passRate,
-        ]),
+        Object.entries(entry.casePassRates).flatMap(([caseId, passRate]) => {
+          if (!hasScoreEvidence(experiment, split))
+            return [[`${entry.profileId}:${caseId}`, passRate]];
+          const sample = entry.caseSamples?.[caseId];
+          const score = sample?.metrics.score;
+          return score?.n === entry.repetitions && sample?.scoreBasis
+            ? [[JSON.stringify([entry.profileId, caseId, sample.scoreBasis]), score.mean]]
+            : [];
+        }),
       ),
   );
 };
@@ -308,8 +428,9 @@ const dominates = (a: Record<string, number>, b: Record<string, number>): boolea
   if (keys.size === 0) return false;
   let strictlyBetter = false;
   for (const key of keys) {
-    const av = a[key] ?? 0;
-    const bv = b[key] ?? 0;
+    const av = a[key];
+    const bv = b[key];
+    if (av === undefined || bv === undefined) return false;
     if (av < bv) return false;
     if (av > bv) strictlyBetter = true;
   }
@@ -361,8 +482,13 @@ const frontierCoverage = (
   const candidateVector = candidateCaseVector(candidate, experiment, "train");
   const keys = new Set(vectors.flatMap((vector) => Object.keys(vector)));
   return [...keys].filter((key) => {
-    const best = Math.max(...vectors.map((vector) => vector[key] ?? 0));
-    return (candidateVector[key] ?? 0) === best;
+    const values = vectors.flatMap((vector) => (vector[key] === undefined ? [] : [vector[key]]));
+    const best = Math.max(...values);
+    return (
+      (frontier.length === 1 || values.length > 1) &&
+      candidateVector[key] !== undefined &&
+      candidateVector[key] === best
+    );
   }).length;
 };
 
@@ -379,6 +505,36 @@ export const searchFrontier = (
   });
 };
 
+const relativeConstraintReport = (
+  campaign: ExperimentCampaign,
+  candidate: ExperimentDecisionCandidate,
+  constraint: ExperimentConstraint,
+  profile: string | undefined,
+  experiment: OpenedExperimentRun["manifest"],
+): ExperimentSelection["constraints"][number] => {
+  const effect = relativeConstraintEffect(candidate, constraint, profile, experiment);
+  const reference = candidate.id === "baseline";
+  let values = effect.meanDelta === null ? [] : [effect.meanDelta];
+  if (reference) values = [0];
+  const note = reference ? "baseline reference; relative gate does not apply" : effect.note;
+  return {
+    candidateId: candidate.id,
+    metric: constraint.metric,
+    relativeTo: "baseline",
+    ...(profile ? { profile } : {}),
+    split: constraint.split,
+    values,
+    passed: satisfiesConstraint(
+      campaign,
+      candidate,
+      { ...constraint, ...(profile ? { profile } : {}) },
+      experiment,
+    ),
+    ...(!reference && effect.interval ? { interval: effect.interval } : {}),
+    ...(note ? { note } : {}),
+  };
+};
+
 export const deriveExperimentSelection = (
   campaign: ExperimentCampaign,
   candidates: readonly ExperimentDecisionCandidate[],
@@ -405,6 +561,9 @@ export const deriveExperimentSelection = (
           ? [constraint.profile]
           : campaign.profiles;
       return profiles.map((profile) => {
+        if (constraint.relativeTo) {
+          return relativeConstraintReport(campaign, candidate, constraint, profile, experiment);
+        }
         const values = metricValues(
           candidate,
           constraint.metric,
@@ -426,6 +585,7 @@ export const deriveExperimentSelection = (
           if (samples) {
             try {
               interval = bootstrapMeanConfidenceInterval(samples, constraint.confidence);
+              note = confidenceGateIssue(interval);
             } catch (error) {
               note = error instanceof Error ? error.message : String(error);
             }

package/src/orchestrator/experiment.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import type { z } from "zod";
+import { independentClusterCount } from "../core/evaluation-clusters.js";
 import {
   bootstrapMeanConfidenceInterval,
   type Candidate,
@@ -47,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -55,14 +57,7 @@ import {
   openExperimentRun,
   resumeExperimentRun,
 } from "./experiment-runs.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  LUX_VERSION,
-  runtimeCliIdentityHashForReferences,
-} from "./experiment-runtime-identity.js";
+import { LUX_VERSION, runtimeCliIdentityHashForReferences } from "./experiment-runtime-identity.js";
 import {
   candidateEvaluations,
   type ExperimentSelectionDerivationOptionsSchema,
@@ -188,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -198,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({
@@ -254,50 +246,57 @@ export const planExperimentCampaign = async (
     const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
     let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
     if (corpus.kind === "grader" && splitSelector) {
-      const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+      let selected = selectGraderAnnotations(graderAnnotations, splitSelector);
       if (constraint.metric === "grader.macroFalsePositiveRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(false),
-        ).length;
+        );
       } else if (constraint.metric === "grader.macroFalseNegativeRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(true),
-        ).length;
+        );
       } else if (constraint.metric === "grader.scoreMae") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.caseScore !== undefined ||
             Object.values(annotation.expected.assertions).some(
               (label) => label.score !== undefined,
             ),
-        ).length;
+        );
       } else if (constraint.metric === "grader.caseAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             annotation.expected.caseScore !== undefined,
-        ).length;
+        );
       } else if (constraint.metric === "grader.assertionAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) => Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       } else if (
         constraint.metric === "grader.abstentionRate" ||
+        constraint.metric === "grader.insufficientEvidenceRate" ||
+        constraint.metric === "grader.operationalErrorRate" ||
         constraint.metric === "grader.coverage"
       ) {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       }
+      examples = independentClusterCount(
+        selected.map((annotation) => ({ id: annotation.id, cluster: annotation.meta?.cluster })),
+      );
     }
+    if (corpus.kind === "agent" && splitSelector)
+      examples = independentClusterCount(selectCases(corpus.cases, splitSelector));
     if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
       examples = 1;
     }
@@ -318,6 +317,10 @@ export const planExperimentCampaign = async (
         Array<number>(examples).fill(0),
         constraint.confidence,
       );
+      if (constraint.relativeTo) {
+        bestForMin.lower = 2 * bestForMin.lower - 1;
+        bestForMax.upper = 2 * bestForMax.upper - 1;
+      }
       if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
         notes.push(
           `${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`,
@@ -986,6 +989,3 @@ export const optimizeExperimentCampaign = async (
     new CampaignExecution(execution, "optimize", runOptimizationCampaignWork),
   );
 };
-
-// Internal exports for identity contract tests.
-export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };

package/src/orchestrator/fixture-identity.ts

@@ -5,6 +5,7 @@ import { contentHash, type FixturesRef, valueHash } from "../core/index.js";
 export type FixtureIdentity = {
   rootDir: string;
   hash: string;
+  fileHashes: ReadonlyMap<string, string>;
 };
 
 const fixtureEntries = async (
@@ -63,5 +64,12 @@ export const fixtureIdentity = async (
   if (metadata.isSymbolicLink() || !metadata.isDirectory()) {
     throw new Error(`fixtures: expected a real directory: ${rootDir}`);
   }
-  return { rootDir, hash: valueHash(await fixtureEntries(rootDir)) };
+  const entries = await fixtureEntries(rootDir);
+  return {
+    rootDir,
+    hash: valueHash(entries),
+    fileHashes: new Map(
+      entries.flatMap((entry) => (entry.hash === undefined ? [] : [[entry.path, entry.hash]])),
+    ),
+  };
 };

package/src/orchestrator/fs-read.ts

@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs";
 import type { FileHandle } from "node:fs/promises";
 import { readdir, readFile } from "node:fs/promises";
-import { join, relative, resolve } from "node:path";
+import { join, relative } from "node:path";
 import type { z } from "zod";
 
 const errorMessage = (error: unknown): string =>
@@ -93,34 +93,4 @@ export const walkFiles = async (root: string, maxFiles?: number): Promise<string
   return out.sort();
 };
 
-/**
- * Walk authored trees without silently following or omitting a symlink that
- * can change outside the experiment identity. Exact ignored paths support
- * generated state directories whose names are project-configurable.
- */
-export const walkAuthoredFiles = async (
-  root: string,
-  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
-  ignoredPaths: ReadonlySet<string> = new Set(),
-): Promise<string[]> => {
-  if (!existsSync(root)) return [];
-  const out: string[] = [];
-  const stack = [root];
-  while (stack.length > 0) {
-    const dir = stack.pop();
-    if (!dir) continue;
-    for (const entry of await readdir(dir, { withFileTypes: true })) {
-      const full = join(dir, entry.name);
-      if (ignoredPaths.has(resolve(full))) continue;
-      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
-      if (entry.isSymbolicLink()) {
-        throw new Error(
-          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
-        );
-      }
-      if (entry.isDirectory()) stack.push(full);
-      else if (entry.isFile()) out.push(relative(root, full));
-    }
-  }
-  return out.sort();
-};
+export { walkAuthoredFiles } from "../core/file-tree.js";