@prismatic-io/lux 0.0.2-preview.40 → 0.0.2-preview.41

package/src/orchestrator/experiment-corpus.ts

@@ -10,6 +10,7 @@ import {
   type ExperimentSelector,
   valueHash,
 } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
 import { experimentDriverIdentity } from "./driver-identity.js";
@@ -74,7 +75,10 @@ export const caseRubricCliReferences = (
   );
   return execution.loaded.campaign.profiles.flatMap((profileId) => {
     const profile = profileOrThrow(execution.config, profileId);
-    const harness = profile.harness ?? execution.config.harness;
+    const harness = resolveHarnessForDriver(
+      profile.harness ?? execution.config.harness,
+      resolveDriverReference(authored, profile.driver, undefined, execution.config.driverDefaults),
+    );
     return rubrics.map(({ index, config }) => {
       const provider = config.provider ?? harness?.provider ?? "claude-code";
       const inheritsHarness =
@@ -129,7 +133,6 @@ export const agentCorpusHash = async (
       const profileCliIdentities = await Promise.all(
         execution.loaded.campaign.profiles.map(async (profileId) => {
           const profile = profileOrThrow(execution.config, profileId);
-          const harness = profile.harness ?? execution.config.harness;
           const driver = bindSubjectRoot(
             resolveDriverReference(
               bound,
@@ -139,6 +142,10 @@ export const agentCorpusHash = async (
             ),
             execution.loaded.subjectRoot,
           );
+          const harness = resolveHarnessForDriver(
+            profile.harness ?? execution.config.harness,
+            driver,
+          );
           const answerer = bindSubjectRoot(
             resolveAnswererReference(
               bound,
@@ -223,6 +230,7 @@ export const graderCorpusHash = async (
           runDir: annotation.runDir,
           evidenceHash: valueHash({
             caseId: run.case.id,
+            driver: run.metadata.driver.name,
             prompt: run.case.prompt,
             events: run.events.flatMap((event) => {
               if (event.type === "lifecycle" || event.type === "ready") return [];

package/src/orchestrator/experiment-identity.ts

@@ -1,7 +1,9 @@
 import { readFile } from "node:fs/promises";
 import { isAbsolute, relative, resolve, sep } from "node:path";
 import { bindSubjectRoot, contentHash, valueHash } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
 import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
 import { authoredDependencyGraphHash } from "./authored-dependencies.js";
 import { assertValidationArgumentsBound } from "./candidates.js";
 import { selectCases } from "./discover.js";
@@ -25,6 +27,8 @@ import {
   resolveDriverReference,
 } from "./orchestrator.js";
 import { runtimeIdentity } from "./provenance.js";
+import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
+import { loadRubricDefinition } from "./rubric-loader.js";
 import { validatePluginConfig } from "./validate-plugin-config.js";
 
 export const resolvedProfileSnapshot = async (
@@ -41,18 +45,24 @@ export const resolvedProfileSnapshot = async (
           execution.loaded.campaign.evaluation.kind === "agent"
             ? (profile.driver ?? execution.config.defaultDriver)
             : undefined;
-        const harness = profile.harness ?? execution.config.harness;
+        const harness = resolveHarnessForDriver(
+          profile.harness ?? execution.config.harness,
+          driver,
+        );
         const answerer =
           execution.loaded.campaign.evaluation.kind === "agent"
-            ? (profile.answerer ?? execution.config.defaultAnswerer)
+            ? resolveAnswererReference(
+                { prompt: "" },
+                profile.answerer ?? execution.config.defaultAnswerer,
+                harness,
+              )
             : undefined;
-        const answererCli = referencedAnswererCli(answerer);
+        const answererCli = driver || harness ? referencedAnswererCli(answerer) : undefined;
         let harnessCommand = harness?.command;
         if (!harnessCommand) {
           harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
         }
-        const capturesHarnessIdentity =
-          Boolean(harness) || execution.loaded.campaign.evaluation.kind === "grader";
+        const capturesHarnessIdentity = Boolean(harness);
         const harnessCliIdentity = capturesHarnessIdentity
           ? await cliIdentity(harnessCommand, execution.config.rootDir)
           : null;
@@ -177,13 +187,28 @@ export const runtimeCliReferences = async (
     );
   }
   if (corpus.kind === "grader") {
-    for (const profileId of execution.loaded.campaign.profiles) {
-      const profile = profileOrThrow(execution.config, profileId);
-      const harness = profile.harness ?? execution.config.harness;
-      add(
-        harness?.command ?? (harness?.provider === "codex" ? "codex" : "claude"),
-        execution.config.rootDir,
-      );
+    const annotations = execution.loaded.graderAnnotations?.annotations ?? [];
+    const selected = new Map(
+      selectorsBySplit(execution.loaded.campaign)
+        .flatMap((split) => selectGraderAnnotations(annotations, split.selector))
+        .map((annotation) => [annotation.id, annotation]),
+    );
+    for (const annotation of selected.values()) {
+      const run = await loadFrozenRegradableRun(annotation.runDir);
+      try {
+        const rubric = await loadRubricDefinition(
+          resolve(execution.loaded.subjectRoot, annotation.rubricPath),
+        );
+        const authored = {
+          ...run.case,
+          assertions: rubric.assertions,
+          driver: run.metadata.driver.name,
+        };
+        for (const reference of caseRubricCliReferences(execution, authored))
+          add(reference.command, execution.config.rootDir);
+      } finally {
+        await releaseFrozenRegradableRun(run);
+      }
     }
   }
   if (corpus.kind === "agent") {
@@ -201,7 +226,6 @@ export const runtimeCliReferences = async (
       );
       for (const profileId of execution.loaded.campaign.profiles) {
         const profile = profileOrThrow(execution.config, profileId);
-        const harness = profile.harness ?? execution.config.harness;
         const driver = bindSubjectRoot(
           resolveDriverReference(
             authored,
@@ -211,6 +235,10 @@ export const runtimeCliReferences = async (
           ),
           execution.loaded.subjectRoot,
         );
+        const harness = resolveHarnessForDriver(
+          profile.harness ?? execution.config.harness,
+          driver,
+        );
         const answerer = bindSubjectRoot(
           resolveAnswererReference(
             authored,

package/src/orchestrator/orchestrator.ts

@@ -23,6 +23,7 @@ import {
   type SuiteLifecycleHookContext,
   valueHash,
 } from "../core/index.js";
+import { resolveHarnessForDriver } from "../core/model-cli.js";
 import {
   mergeAnswererReference,
   mergeDriverReference,
@@ -374,13 +375,9 @@ export class Orchestrator {
       request.subjectRoot,
     );
     const driverName = driverNameOf(driverRef);
+    const harness = resolveHarnessForDriver(this.harness, driverRef);
     const answererRef = bindSubjectRoot(
-      resolveAnswererReference(
-        evalCase,
-        this.defaultAnswerer,
-        this.harness,
-        request.answererOverride,
-      ),
+      resolveAnswererReference(evalCase, this.defaultAnswerer, harness, request.answererOverride),
       request.subjectRoot,
     );
     const answererName = answererNameOf(answererRef);
@@ -445,7 +442,7 @@ export class Orchestrator {
         implementationHash,
         assertions: assertionImplementations,
         validatedAssertionsHash,
-        graderContext: this.harness,
+        graderContext: harness,
         skipJudge: request.skipJudge ?? false,
       });
       return await this.runWithPlugins(
@@ -467,7 +464,7 @@ export class Orchestrator {
           evaluatorConfigHash: valueHash({
             luxVersion: LUX_VERSION,
             implementationHash,
-            harness: this.harness,
+            harness,
             driver: { name: driverPlugin.name, create: driverPlugin.create },
             answerer: { name: answererPlugin.name, create: answererPlugin.create },
             assertions: assertionImplementations,

package/src/orchestrator/regrade.ts

@@ -33,6 +33,7 @@ import {
   readRunArtifactBytes,
   valueHash,
 } from "../core/index.js";
+import { HarnessModelConfigSchema, resolveHarnessForDriver } from "../core/model-cli.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import { parsePersistedJson, readBoundedHandle, safeReadJson } from "./fs-read.js";
@@ -539,12 +540,15 @@ export const regradeRun = async (options: {
     const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
     const assertionEvaluators = assertionEvaluatorIdentities(run.case.assertions ?? [], registry);
     const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], registry);
+    const configuredHarness = HarnessModelConfigSchema.safeParse(options.graderContext);
     const graderHash = graderConfigHash({
       luxVersion: LUX_VERSION,
       implementationHash: await currentEvaluatorImplementationHash(),
       assertions: assertionEvaluators,
       validatedAssertionsHash,
-      graderContext: options.graderContext,
+      graderContext: configuredHarness.success
+        ? configuredHarness.data
+        : (options.graderContext ?? resolveHarnessForDriver(undefined, run.metadata.driver)),
     });
     const report: GradingReport = await gradeRun(
       run,