@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/experiment.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import type { z } from "zod";
+import { independentClusterCount } from "../core/evaluation-clusters.js";
 import {
   bootstrapMeanConfidenceInterval,
   type Candidate,
@@ -47,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -55,14 +57,7 @@ import {
   openExperimentRun,
   resumeExperimentRun,
 } from "./experiment-runs.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  LUX_VERSION,
-  runtimeCliIdentityHashForReferences,
-} from "./experiment-runtime-identity.js";
+import { LUX_VERSION, runtimeCliIdentityHashForReferences } from "./experiment-runtime-identity.js";
 import {
   candidateEvaluations,
   type ExperimentSelectionDerivationOptionsSchema,
@@ -188,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -198,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({
@@ -254,50 +246,57 @@ export const planExperimentCampaign = async (
     const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
     let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
     if (corpus.kind === "grader" && splitSelector) {
-      const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+      let selected = selectGraderAnnotations(graderAnnotations, splitSelector);
       if (constraint.metric === "grader.macroFalsePositiveRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(false),
-        ).length;
+        );
       } else if (constraint.metric === "grader.macroFalseNegativeRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(true),
-        ).length;
+        );
       } else if (constraint.metric === "grader.scoreMae") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.caseScore !== undefined ||
             Object.values(annotation.expected.assertions).some(
               (label) => label.score !== undefined,
             ),
-        ).length;
+        );
       } else if (constraint.metric === "grader.caseAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             annotation.expected.caseScore !== undefined,
-        ).length;
+        );
       } else if (constraint.metric === "grader.assertionAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) => Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       } else if (
         constraint.metric === "grader.abstentionRate" ||
+        constraint.metric === "grader.insufficientEvidenceRate" ||
+        constraint.metric === "grader.operationalErrorRate" ||
         constraint.metric === "grader.coverage"
       ) {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       }
+      examples = independentClusterCount(
+        selected.map((annotation) => ({ id: annotation.id, cluster: annotation.meta?.cluster })),
+      );
     }
+    if (corpus.kind === "agent" && splitSelector)
+      examples = independentClusterCount(selectCases(corpus.cases, splitSelector));
     if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
       examples = 1;
     }
@@ -318,6 +317,10 @@ export const planExperimentCampaign = async (
         Array<number>(examples).fill(0),
         constraint.confidence,
       );
+      if (constraint.relativeTo) {
+        bestForMin.lower = 2 * bestForMin.lower - 1;
+        bestForMax.upper = 2 * bestForMax.upper - 1;
+      }
       if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
         notes.push(
           `${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`,
@@ -986,6 +989,3 @@ export const optimizeExperimentCampaign = async (
     new CampaignExecution(execution, "optimize", runOptimizationCampaignWork),
   );
 };
-
-// Internal exports for identity contract tests.
-export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };

package/src/orchestrator/fixture-identity.ts

@@ -5,6 +5,7 @@ import { contentHash, type FixturesRef, valueHash } from "../core/index.js";
 export type FixtureIdentity = {
   rootDir: string;
   hash: string;
+  fileHashes: ReadonlyMap<string, string>;
 };
 
 const fixtureEntries = async (
@@ -63,5 +64,12 @@ export const fixtureIdentity = async (
   if (metadata.isSymbolicLink() || !metadata.isDirectory()) {
     throw new Error(`fixtures: expected a real directory: ${rootDir}`);
   }
-  return { rootDir, hash: valueHash(await fixtureEntries(rootDir)) };
+  const entries = await fixtureEntries(rootDir);
+  return {
+    rootDir,
+    hash: valueHash(entries),
+    fileHashes: new Map(
+      entries.flatMap((entry) => (entry.hash === undefined ? [] : [[entry.path, entry.hash]])),
+    ),
+  };
 };

package/src/orchestrator/fs-read.ts

@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs";
 import type { FileHandle } from "node:fs/promises";
 import { readdir, readFile } from "node:fs/promises";
-import { join, relative, resolve } from "node:path";
+import { join, relative } from "node:path";
 import type { z } from "zod";
 
 const errorMessage = (error: unknown): string =>
@@ -93,34 +93,4 @@ export const walkFiles = async (root: string, maxFiles?: number): Promise<string
   return out.sort();
 };
 
-/**
- * Walk authored trees without silently following or omitting a symlink that
- * can change outside the experiment identity. Exact ignored paths support
- * generated state directories whose names are project-configurable.
- */
-export const walkAuthoredFiles = async (
-  root: string,
-  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
-  ignoredPaths: ReadonlySet<string> = new Set(),
-): Promise<string[]> => {
-  if (!existsSync(root)) return [];
-  const out: string[] = [];
-  const stack = [root];
-  while (stack.length > 0) {
-    const dir = stack.pop();
-    if (!dir) continue;
-    for (const entry of await readdir(dir, { withFileTypes: true })) {
-      const full = join(dir, entry.name);
-      if (ignoredPaths.has(resolve(full))) continue;
-      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
-      if (entry.isSymbolicLink()) {
-        throw new Error(
-          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
-        );
-      }
-      if (entry.isDirectory()) stack.push(full);
-      else if (entry.isFile()) out.push(relative(root, full));
-    }
-  }
-  return out.sort();
-};
+export { walkAuthoredFiles } from "../core/file-tree.js";

package/src/orchestrator/grader-experiment.ts

@@ -1,10 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   type Candidate,
   type CandidateEvaluation,
@@ -28,6 +25,7 @@ import {
 import type { CampaignExecutionOptions } from "./experiment.js";
 import type { OpenedExperimentRun } from "./experiment-runs.js";
 import { gradeRun } from "./grading.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
 import { loadRubricDefinition } from "./rubric-loader.js";
 
@@ -38,10 +36,12 @@ const GraderJudgmentSchema = z.object({
   actual: z.union([z.boolean(), z.number()]).nullable(),
   matched: z.boolean(),
   absoluteError: z.exactOptional(z.number()),
+  unavailableReason: z.exactOptional(z.enum(["insufficient-evidence", "operational-error"])),
 });
 export type GraderJudgment = z.infer<typeof GraderJudgmentSchema>;
 
 const GraderEvidenceItemSchema = z.object({
+  clusterId: z.exactOptional(z.string().min(1)),
   annotationId: z.string(),
   repetition: z.number().int().positive(),
   runDir: z.string(),
@@ -95,6 +95,11 @@ const assertionOperationalError = (
   return detail.success && detail.data.graderError !== undefined;
 };
 
+const assertionUnavailableReason = (
+  entry: GradingReport["assertions"][number] | undefined,
+): GraderJudgment["unavailableReason"] =>
+  assertionOperationalError(entry) ? "operational-error" : entry?.abstention;
+
 const assertionModelAccounting = (
   assertion: GradingReport["assertions"][number],
 ): { attempts: number; costUsd: number; unpricedModelCalls: number } => {
@@ -115,19 +120,22 @@ export const compareGraderAnnotation = (
   scoreTolerance: number,
 ): GraderJudgment[] => {
   const judgments: GraderJudgment[] = [];
-  const reportFailed = report.assertions.some(assertionOperationalError);
+  const reportUnavailableReason = report.assertions.some(assertionOperationalError)
+    ? "operational-error"
+    : report.assertions.find((entry) => entry.abstention)?.abstention;
   if (annotation.expected.casePassed !== undefined) {
-    const actual = reportFailed ? null : report.casePassed;
+    const actual = reportUnavailableReason ? null : report.casePassed;
     judgments.push({
       key: "case",
       kind: "case-pass",
       expected: annotation.expected.casePassed,
       actual,
       matched: actual === annotation.expected.casePassed,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   if (annotation.expected.caseScore !== undefined) {
-    const actual = reportFailed ? null : report.caseScore;
+    const actual = reportUnavailableReason ? null : report.caseScore;
     const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
     judgments.push({
       key: "case",
@@ -136,6 +144,7 @@ export const compareGraderAnnotation = (
       actual,
       matched: actual !== null && absoluteError <= scoreTolerance,
       absoluteError,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   const entries = new Map(
@@ -143,13 +152,15 @@ export const compareGraderAnnotation = (
   );
   for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
     const entry = entries.get(assertionId);
-    const actual = assertionOperationalError(entry) ? undefined : entry;
+    const unavailableReason = assertionUnavailableReason(entry);
+    const actual = unavailableReason ? undefined : entry;
     judgments.push({
       key: assertionId,
       kind: "assertion-pass",
       expected: expected.passed,
       actual: actual?.passed ?? null,
       matched: actual?.passed === expected.passed,
+      ...(unavailableReason ? { unavailableReason } : {}),
     });
     if (expected.score !== undefined) {
       const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
@@ -160,6 +171,7 @@ export const compareGraderAnnotation = (
         actual: actual?.score ?? null,
         matched: actual !== undefined && absoluteError <= scoreTolerance,
         absoluteError,
+        ...(unavailableReason ? { unavailableReason } : {}),
       });
     }
   }
@@ -233,7 +245,7 @@ const collectGraderEvidence = (
 
 export const summarizeGraderEvidence = (
   items: readonly z.infer<typeof GraderEvidenceItemSchema>[],
-  candidate: Candidate,
+  candidate: Pick<Candidate, "sourceBytes">,
 ): { metrics: Record<string, number>; samples: Record<string, number[]> } => {
   const samples: Record<string, number[]> = {};
   const { byAnnotation, itemsByAnnotation, counts } = collectGraderEvidence(items);
@@ -323,6 +335,18 @@ export const summarizeGraderEvidence = (
         "grader.abstentionRate",
         passes.filter((judgment) => judgment.actual === null).length / passes.length,
       );
+      addSample(
+        samples,
+        "grader.insufficientEvidenceRate",
+        passes.filter((judgment) => judgment.unavailableReason === "insufficient-evidence").length /
+          passes.length,
+      );
+      addSample(
+        samples,
+        "grader.operationalErrorRate",
+        passes.filter((judgment) => judgment.unavailableReason === "operational-error").length /
+          passes.length,
+      );
     }
   }
   const precision = ratio(
@@ -440,7 +464,42 @@ const evaluationFromEvidence = (
   evidencePath: string,
   evidence: GraderEvaluationEvidence,
 ): CandidateEvaluation & { evidencePath: string } => {
+  const caseSamples = clusterCaseSamples(
+    evidence.items.map((item) => {
+      const summary = summarizeGraderEvidence([item], {
+        sourceBytes: evidence.metrics["subject.bytes"] ?? 0,
+      });
+      return {
+        [item.annotationId]: {
+          ...(item.clusterId ? { clusterId: item.clusterId } : {}),
+          identityHash: experiment.manifest.evaluationHash
+            ? valueHash({
+                evaluationHash: experiment.manifest.evaluationHash,
+                annotationId: item.annotationId,
+                profileId: evidence.profileId,
+                split: evidence.split,
+              })
+            : null,
+          metrics: Object.fromEntries(
+            Object.entries(summary.samples).flatMap(([metric, values]) =>
+              values.length === 1 ? [[metric, { mean: values[0] ?? 0, n: 1 }]] : [],
+            ),
+          ),
+        },
+      };
+    }),
+  );
   const byAnnotation = new Map<string, number[]>();
+  const grouped = groupedCaseMetricSamples(caseSamples, evidence.repetitions);
+  const metrics = grouped
+    ? Object.fromEntries(
+        Object.entries(evidence.metrics).flatMap(([name, value]) => {
+          if (!Object.hasOwn(evidence.samples, name)) return [[name, value]];
+          const values = grouped[name];
+          return values ? [[name, mean(values)]] : [];
+        }),
+      )
+    : evidence.metrics;
   for (const item of evidence.items) {
     const values = byAnnotation.get(item.annotationId) ?? [];
     values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
@@ -471,10 +530,11 @@ const evaluationFromEvidence = (
     profileId: evidence.profileId,
     split: evidence.split,
     evidencePath: relative(experiment.path, evidencePath),
-    passRate: evidence.metrics["quality.passRate"] ?? 0,
+    passRate: metrics["quality.passRate"] ?? 0,
     casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
-    metrics: evidence.metrics,
-    samples: evidence.samples,
+    metrics,
+    samples: grouped ?? evidence.samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: evidence.items.length,
     judgeCalls: evidence.items.reduce(
       (total, item) =>
@@ -648,7 +708,7 @@ export const evaluateGraderCandidate = async (options: {
   const registry = buildRegistryFromConfig({
     ...execution.config,
     ...(harness ? { harness } : {}),
-  }).withAssertion(createRubricAssertion(harness));
+  });
   const items: GraderEvidenceItem[] = [];
   for (const annotation of selected) {
     const rubric = await loadRubricDefinition(
@@ -779,6 +839,7 @@ export const evaluateGraderCandidate = async (options: {
         );
         const item: GraderEvidenceItem = {
           annotationId: annotation.id,
+          ...(annotation.meta?.cluster ? { clusterId: annotation.meta.cluster } : {}),
           repetition,
           runDir: annotation.runDir,
           rubricPath: annotation.rubricPath,

package/src/orchestrator/grading.ts

@@ -3,6 +3,7 @@ import {
   type Assertion,
   type AssertionImpl,
   type AssertionProgress,
+  type AssertionResult,
   AssertionResultSchema,
   type GradingAssertionEntry,
   type GradingReport,
@@ -14,7 +15,7 @@ import {
 } from "../core/index.js";
 import { validatePluginConfigAsync } from "./validate-plugin-config.js";
 
-const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold", "informational"]);
 
 const shapeOf = (schema: z.ZodTypeAny): Record<string, unknown> | null => {
   const shape = (schema as { shape?: unknown }).shape;
@@ -124,6 +125,56 @@ const describeAssertion = (a: Assertion): string => {
   return fields ? `${a.type} ${fields}` : a.type;
 };
 
+const assertionPassed = (result: AssertionResult, threshold: number): boolean =>
+  result.errorMessage === undefined &&
+  result.abstention === undefined &&
+  result.passed &&
+  result.score >= threshold;
+
+export const gradingScoreBasis = (report: GradingReport | null): string | null =>
+  report?.caseScore === null || !report
+    ? null
+    : valueHash(
+        report.assertions
+          .map((entry, index) => ({ entry, id: entry.id ?? `${entry.type}:${index + 1}` }))
+          .filter(({ entry }) => entry.errorMessage === undefined && entry.abstention === undefined)
+          .map(({ entry, id }) => ({ id, weight: entry.weight }))
+          .sort((a, b) => a.id.localeCompare(b.id)),
+      );
+
+const aggregateGrading = (
+  entries: GradingAssertionEntry[],
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>,
+  gradingIncomplete: boolean,
+): GradingReport => {
+  const requiredEntries = entries.filter((entry) => entry.passed !== null);
+  const incomplete =
+    gradingIncomplete || requiredEntries.some((entry) => entry.errorMessage !== undefined);
+  const gradingAbstained = requiredEntries.some((entry) => entry.abstention !== undefined);
+  const scoredEntries = entries.filter(
+    (entry) => entry.errorMessage === undefined && entry.abstention === undefined,
+  );
+  const weightSum = scoredEntries.reduce((sum, entry) => sum + entry.weight, 0);
+  const weightedScoreSum = scoredEntries.reduce(
+    (sum, entry) => sum + entry.weight * entry.score,
+    0,
+  );
+  const hasVerdict = !incomplete && !gradingAbstained && weightSum > 0;
+  return {
+    schemaVersion: 1,
+    casePassed:
+      hasVerdict && requiredEntries.length > 0
+        ? requiredEntries.every((entry) => entry.passed)
+        : null,
+    caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
+    assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
+    ...(incomplete
+      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
+      : {}),
+  };
+};
+
 export const gradeRun = async (
   run: Run,
   registry: Registry,
@@ -150,9 +201,6 @@ export const gradeRun = async (
     name: string;
     reason: "judge-disabled";
   }> = [];
-  let weightedScoreSum = 0;
-  let weightSum = 0;
-  let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
@@ -205,7 +253,7 @@ export const gradeRun = async (
       if (validatedConfigs && index >= validatedConfigs.length) {
         throw new Error(`validated assertion plan is missing assertions[${index}]`);
       }
-      // Pass the validated config so schema defaults (e.g. timeoutMs)
+      // Pass the validated config so schema defaults (e.g. timeout)
       // reach evaluate(); the raw assertion object has none applied.
       const result = AssertionResultSchema.parse(
         await impl.evaluate(validated as typeof a, run, {
@@ -219,43 +267,35 @@ export const gradeRun = async (
             : {}),
         }),
       );
-      const passed =
-        result.errorMessage === undefined && result.passed && result.score >= threshold;
+      const passed = assertionPassed(result, threshold);
       entry = {
         id,
         type: a.type,
         name,
-        passed,
+        passed: a.informational ? null : passed,
+        ...(a.informational ? { informational: true } : { threshold }),
         score: result.score,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         ...(result.comment !== undefined ? { comment: result.comment } : {}),
         detail: result.detail,
+        ...(result.abstention !== undefined ? { abstention: result.abstention } : {}),
         ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
         ...(result.usage ? { usage: result.usage } : {}),
       };
-      if (result.errorMessage !== undefined) {
-        gradingIncomplete = true;
-      } else {
-        if (!passed) allPassed = false;
-        weightedScoreSum += weight * result.score;
-        weightSum += weight;
-      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       entry = {
         id,
         type: a.type,
         name,
-        passed: false,
+        passed: a.informational ? null : false,
+        ...(a.informational ? { informational: true } : { threshold }),
         score: 0,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         errorMessage: message,
       };
-      gradingIncomplete = true;
     }
 
     entries.push(entry);
@@ -264,16 +304,5 @@ export const gradeRun = async (
     await modelAttempts?.complete(assertionAttemptIds);
   }
 
-  let caseScore: number | null = null;
-  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
-  return {
-    schemaVersion: 1,
-    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
-    caseScore,
-    assertions: entries,
-    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
-    ...(gradingIncomplete
-      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
-      : {}),
-  };
+  return aggregateGrading(entries, skippedAssertions, gradingIncomplete);
 };