@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/experiment-corpus.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import {
   bindSubjectRoot,
   type Candidate,
@@ -9,18 +10,17 @@ import {
   type ExperimentSelector,
   valueHash,
 } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import { experimentDriverIdentity } from "./driver-identity.js";
+import type {
+  CampaignExecutionOptions,
+  CorpusSelection,
+  EvaluationCorpus,
+} from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  referencedAnswererCli,
-  referencedCli,
-} from "./experiment-runtime-identity.js";
+import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
 import { loadEvalCase } from "./loader.js";
 import { resolveAnswererReference, resolveDriverReference } from "./orchestrator.js";
@@ -75,7 +75,10 @@ export const caseRubricCliReferences = (
   );
   return execution.loaded.campaign.profiles.flatMap((profileId) => {
     const profile = profileOrThrow(execution.config, profileId);
-    const harness = profile.harness ?? execution.config.harness;
+    const harness = resolveHarnessForDriver(
+      profile.harness ?? execution.config.harness,
+      resolveDriverReference(authored, profile.driver, undefined, execution.config.driverDefaults),
+    );
     return rubrics.map(({ index, config }) => {
       const provider = config.provider ?? harness?.provider ?? "claude-code";
       const inheritsHarness =
@@ -123,8 +126,6 @@ export const agentCorpusHash = async (
         .map((evalCase) => [evalCase.path, evalCase]),
     ).values(),
   ];
-  const caseBehaviorHashes = new Map<string, Promise<string | null>>();
-  const configuredSourceHashes = new Map<string, Promise<string>>();
   const definitions = await Promise.all(
     selected.map(async (evalCase) => {
       const authored = await loadEvalCase(evalCase.path);
@@ -132,40 +133,35 @@ export const agentCorpusHash = async (
       const profileCliIdentities = await Promise.all(
         execution.loaded.campaign.profiles.map(async (profileId) => {
           const profile = profileOrThrow(execution.config, profileId);
-          const harness = profile.harness ?? execution.config.harness;
-          const driver = resolveDriverReference(
-            bound,
-            profile.driver ?? execution.config.defaultDriver,
+          const driver = bindSubjectRoot(
+            resolveDriverReference(
+              bound,
+              profile.driver ?? execution.config.defaultDriver,
+              undefined,
+              execution.config.driverDefaults,
+            ),
+            execution.loaded.subjectRoot,
           );
-          const answerer = resolveAnswererReference(
-            bound,
-            profile.answerer ?? execution.config.defaultAnswerer,
-            harness,
+          const harness = resolveHarnessForDriver(
+            profile.harness ?? execution.config.harness,
+            driver,
+          );
+          const answerer = bindSubjectRoot(
+            resolveAnswererReference(
+              bound,
+              profile.answerer ?? execution.config.defaultAnswerer,
+              harness,
+            ),
+            execution.loaded.subjectRoot,
           );
-          const driverCli = referencedCli(driver);
           const answererCli = referencedAnswererCli(answerer);
-          const configuredRoot =
-            driverCli?.name === "claude-code" ? claudeConfigRoot(driver) : null;
-          let driverClaudeBehaviorHash: string | null = null;
-          if (configuredRoot) {
-            let hash = caseBehaviorHashes.get(configuredRoot);
-            if (!hash) {
-              hash = claudeUserBehaviorHash(configuredRoot);
-              caseBehaviorHashes.set(configuredRoot, hash);
-            }
-            driverClaudeBehaviorHash = await hash;
-          }
           return {
             profileId,
             driver,
-            driverCliIdentity: driverCli
-              ? await cliIdentity(driverCli.command, execution.config.rootDir)
-              : null,
-            driverClaudeBehaviorHash,
-            configuredDriverSources: await configuredClaudeSourceHashes(
+            driverIdentity: await experimentDriverIdentity(
               driver,
+              execution.registry,
               execution.config.rootDir,
-              configuredSourceHashes,
             ),
             answerer,
             answererCliIdentity: answererCli
@@ -234,6 +230,7 @@ export const graderCorpusHash = async (
           runDir: annotation.runDir,
           evidenceHash: valueHash({
             caseId: run.case.id,
+            driver: run.metadata.driver.name,
             prompt: run.case.prompt,
             events: run.events.flatMap((event) => {
               if (event.type === "lifecycle" || event.type === "ready") return [];
@@ -275,6 +272,21 @@ export const graderCorpusHash = async (
 export const prepareCorpus = async (
   execution: CampaignExecutionOptions,
 ): Promise<EvaluationCorpus> => {
+  const corpus = await inspectCorpus(execution);
+  if (corpus.kind === "grader") return corpus;
+  return {
+    ...corpus,
+    hash: await agentCorpusHash(
+      execution,
+      corpus.cases,
+      selectorsBySplit(execution.loaded.campaign),
+    ),
+  };
+};
+
+export const inspectCorpus = async (
+  execution: CampaignExecutionOptions,
+): Promise<CorpusSelection> => {
   const campaign = execution.loaded.campaign;
   const splits = selectorsBySplit(campaign);
   if (campaign.evaluation.kind === "grader") {
@@ -303,6 +315,24 @@ export const prepareCorpus = async (
     );
   }
   const owners = new Map<string, string>();
+  assertDisjointClusters(
+    splits.map((split) => ({
+      name: split.name,
+      items: selectCases(discovery.cases, split.selector),
+    })),
+  );
+  for (const constraint of campaign.objectives.constraints) {
+    if (constraint.confidence?.method !== "wilson") continue;
+    const selector = splits.find((split) => split.name === constraint.split)?.selector;
+    if (
+      selector &&
+      selectCases(discovery.cases, selector).some((evalCase) => evalCase.cluster !== undefined)
+    ) {
+      throw new Error(
+        "authored case clusters produce bounded means; use Hoeffding or percentile-bootstrap confidence instead of Wilson",
+      );
+    }
+  }
   const caseIds = new Map<string, string>();
   const evidenceOwners = new Map<
     string,
@@ -349,7 +379,6 @@ export const prepareCorpus = async (
   return {
     kind: "agent",
     cases: discovery.cases,
-    hash: await agentCorpusHash(execution, discovery.cases, splits),
   };
 };
 

package/src/orchestrator/experiment-effects.ts

@@ -0,0 +1,63 @@
+import { z } from "zod";
+import { ConfidenceConfigSchema } from "../core/index.js";
+import type { ExperimentRun } from "./experiment-runs.js";
+import { candidateEvaluations } from "./experiment-selection.js";
+import { PairedEffectSchema, pairedEffect } from "./paired-effects.js";
+
+export const ExperimentEffectsSchema = z.array(
+  PairedEffectSchema.extend({
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    metric: z.string(),
+  }),
+);
+
+export const experimentEffects = (
+  experiment: ExperimentRun,
+): z.infer<typeof ExperimentEffectsSchema> => {
+  const candidateIds =
+    experiment.selection?.decisionCandidateIds ??
+    experiment.candidates.map((candidate) => candidate.id);
+  const baseline = candidateEvaluations(experiment, "baseline");
+  const metrics = new Set([
+    "quality.passRate",
+    experiment.evaluationKind === "grader" ? "harness.total" : "agent.total",
+    ...(experiment.evaluationKind === "agent" ? ["score", "interrupts.ask", "toolCalls"] : []),
+    ...experiment.campaignSnapshot.objectives.constraints
+      .filter((constraint) => constraint.relativeTo)
+      .map((constraint) => constraint.metric),
+  ]);
+  return candidateIds
+    .filter((id) => id !== "baseline")
+    .flatMap((candidateId) =>
+      candidateEvaluations(experiment, candidateId).flatMap((after) =>
+        [...metrics].map((metric) => {
+          const before = baseline.find(
+            (entry) => entry.profileId === after.profileId && entry.split === after.split,
+          );
+          const constraint = experiment.campaignSnapshot.objectives.constraints.find(
+            (entry) =>
+              entry.relativeTo &&
+              entry.metric === metric &&
+              entry.split === after.split &&
+              (!entry.profile || entry.profile === after.profileId),
+          );
+          const confidence =
+            constraint?.confidence ??
+            ConfidenceConfigSchema.parse({
+              method: ["quality.passRate", "score"].includes(metric)
+                ? "hoeffding"
+                : "percentile-bootstrap",
+            });
+          return {
+            candidateId,
+            profileId: after.profileId,
+            split: after.split,
+            metric,
+            ...pairedEffect(before, after, metric, confidence),
+          };
+        }),
+      ),
+    );
+};

package/src/orchestrator/experiment-evaluation.ts

@@ -2,10 +2,7 @@ import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   addUsage,
   type Candidate,
@@ -17,6 +14,7 @@ import {
   valueHash,
 } from "../core/index.js";
 import { materializeCandidate } from "./candidates.js";
+import { runComparisonIdentity } from "./comparison-identity.js";
 import { readContentBoundJson, writeContentBoundJson } from "./content-bound-json.js";
 import { type discoverCases, selectCases } from "./discover.js";
 import { ensureBudget, throwIfAborted } from "./experiment-budget.js";
@@ -25,8 +23,10 @@ import { profileOrThrow } from "./experiment-context.js";
 import type { ExperimentEvaluation, OpenedExperimentRun } from "./experiment-runs.js";
 import { average } from "./experiment-selection.js";
 import { evaluateGraderCandidate } from "./grader-experiment.js";
+import { registryWithHarness } from "./harness-registry.js";
 import { loadEvalCase } from "./loader.js";
 import { Orchestrator, type SuiteCaseResult, type SuiteItem } from "./orchestrator.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadSuiteSummary, type SuiteSummary } from "./suite-summary.js";
 
 type ExperimentSplit = "train" | "validation" | "test";
@@ -44,6 +44,7 @@ const gradingFailureMessage = (results: readonly SuiteCaseResult[]): string | un
     }
     const report = result.result?.grading;
     const failedAssertion = report.assertions.find((assertion) => {
+      if (assertion.passed === null) return false;
       if (assertion.errorMessage) return true;
       const detail =
         assertion.detail && typeof assertion.detail === "object"
@@ -101,6 +102,23 @@ const evaluationMetrics = (
   ),
 });
 
+const evaluationMetricDefinitions = (summary: SuiteSummary) =>
+  Object.fromEntries(
+    Object.entries(summary.metrics).flatMap(([name, metric]) => {
+      if (!metric.unit && !metric.label && !metric.better) return [];
+      return [
+        [
+          name,
+          {
+            unit: metric.unit ?? "number",
+            better: metric.better ?? "neutral",
+            ...(metric.label ? { label: metric.label } : {}),
+          },
+        ],
+      ];
+    }),
+  );
+
 const summaryUsage = (summary: SuiteSummary) =>
   summary.samples.reduce(
     (usage, sample) =>
@@ -263,8 +281,38 @@ const agentEvaluationFromSuite = (input: {
   summary: SuiteSummary;
   results: readonly SuiteCaseResult[];
   suiteDir: string;
+  evaluationHash: string | undefined;
+  clusterId: string | undefined;
 }): CandidateEvaluation => {
   const { candidate, profileId, split, summary, results, suiteDir } = input;
+  const caseSamples = clusterCaseSamples(
+    summary.samples.map((sample) => {
+      const metadata = results.find((result) => result.caseId === sample.caseId)?.result?.metadata;
+      // Validated assertion configs contain a fresh candidate workspace path.
+      // The campaign identity binds the authored evaluator without that incidental path.
+      const identityHash =
+        metadata?.provenance && input.evaluationHash
+          ? runComparisonIdentity({
+              ...metadata,
+              provenance: { ...metadata.provenance, evaluatorConfigHash: input.evaluationHash },
+            })
+          : null;
+      return {
+        [sample.canonicalCaseId]: {
+          identityHash,
+          scoreBasis: sample.scoreBasis ?? null,
+          ...(input.clusterId ? { clusterId: input.clusterId } : {}),
+          metrics: Object.fromEntries(
+            Object.entries({
+              ...sample.metrics,
+              ...(sample.score !== undefined ? { score: sample.score } : {}),
+              "quality.passRate": sample.passed === true ? 1 : 0,
+            }).map(([name, value]) => [name, { mean: value, n: 1 }]),
+          ),
+        },
+      };
+    }),
+  );
   const modelAccounting = results.map((result) =>
     modelAccountingFromMetadata(result.result?.metadata ?? null),
   );
@@ -277,7 +325,12 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
+    metricDefinitions: {
+      score: { unit: "score", better: "higher", label: "Quality score" },
+      ...evaluationMetricDefinitions(summary),
+    },
     samples: evaluationSamples(summary),
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
       (total, result) =>
@@ -344,7 +397,7 @@ const aggregateAgentUnits = (
       caseValues.set(evaluatedCaseId, values);
     }
   }
-  const samples = Object.fromEntries(
+  const ungroupedSamples = Object.fromEntries(
     [...samplesByCase].map(([name, byCase]) => [
       name,
       [...byCase.entries()]
@@ -352,6 +405,19 @@ const aggregateAgentUnits = (
         .map(([, values]) => average(values)),
     ]),
   );
+  const metricDefinitions = Object.assign(
+    {},
+    ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
+  );
+  const caseSamples = clusterCaseSamples(units.map((unit) => unit.evaluation.caseSamples));
+  const grouped = groupedCaseMetricSamples(caseSamples, repetitions);
+  const samples = grouped ?? ungroupedSamples;
+  const completeScore =
+    caseSamples &&
+    Object.values(caseSamples).every(
+      (sample) => sample.metrics.score?.n === repetitions && Boolean(sample.scoreBasis),
+    );
+  if (!completeScore) delete samples.score;
   return {
     candidateId: candidate.id,
     profileId,
@@ -362,12 +428,20 @@ const aggregateAgentUnits = (
       [...caseValues].map(([caseId, values]) => [caseId, average(values)]),
     ),
     metrics: Object.fromEntries(
-      [...metricValues].map(([name, values]) => [
-        name,
-        name === "subject.bytes" ? candidate.sourceBytes : average(values),
-      ]),
+      [...metricValues]
+        .filter(([name]) =>
+          name === "score"
+            ? completeScore
+            : !grouped || name === "subject.bytes" || Object.hasOwn(grouped, name),
+        )
+        .map(([name, values]) => [
+          name,
+          name === "subject.bytes" ? candidate.sourceBytes : average(grouped?.[name] ?? values),
+        ]),
     ),
+    metricDefinitions,
     samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
     unpricedModelCalls: units.reduce(
@@ -436,8 +510,10 @@ export const evaluateAgentCandidate = async (options: {
   const profile = profileOrThrow(execution.config, profileId);
   const harness = profile.harness ?? execution.config.harness;
   const defaultDriver = profile.driver ?? execution.config.defaultDriver;
-  const registry = execution.registry.withAssertion(createRubricAssertion(harness));
+  const registry = registryWithHarness(execution.registry, execution.config, harness);
   const orchestrator = new Orchestrator({
+    driverDefaults: execution.config.driverDefaults,
+    environment: profile.environment ?? execution.config.environment,
     registry,
     defaultAnswerer: profile.answerer ?? execution.config.defaultAnswerer,
     ...(defaultDriver ? { defaultDriver } : {}),
@@ -606,6 +682,8 @@ export const evaluateAgentCandidate = async (options: {
             summary,
             results: suite.results,
             suiteDir: suite.suiteDir,
+            evaluationHash: experiment.manifest.evaluationHash,
+            clusterId: evalCase.cluster,
           });
         } finally {
           await rm(unitRoot, { recursive: true, force: true });