@prismatic-io/lux 0.0.2-preview.26 → 0.0.2-preview.28

package/src/orchestrator/experiment-evaluation.ts

@@ -2,10 +2,7 @@ import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   addUsage,
   type Candidate,
@@ -26,6 +23,7 @@ import { profileOrThrow } from "./experiment-context.js";
 import type { ExperimentEvaluation, OpenedExperimentRun } from "./experiment-runs.js";
 import { average } from "./experiment-selection.js";
 import { evaluateGraderCandidate } from "./grader-experiment.js";
+import { registryWithHarness } from "./harness-registry.js";
 import { loadEvalCase } from "./loader.js";
 import { Orchestrator, type SuiteCaseResult, type SuiteItem } from "./orchestrator.js";
 import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
@@ -496,7 +494,7 @@ export const evaluateAgentCandidate = async (options: {
   const profile = profileOrThrow(execution.config, profileId);
   const harness = profile.harness ?? execution.config.harness;
   const defaultDriver = profile.driver ?? execution.config.defaultDriver;
-  const registry = execution.registry.withAssertion(createRubricAssertion(harness));
+  const registry = registryWithHarness(execution.registry, execution.config, harness);
   const orchestrator = new Orchestrator({
     environment: profile.environment ?? execution.config.environment,
     registry,

package/src/orchestrator/grader-experiment.ts

@@ -1,10 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   type Candidate,
   type CandidateEvaluation,
@@ -711,7 +708,7 @@ export const evaluateGraderCandidate = async (options: {
   const registry = buildRegistryFromConfig({
     ...execution.config,
     ...(harness ? { harness } : {}),
-  }).withAssertion(createRubricAssertion(harness));
+  });
   const items: GraderEvidenceItem[] = [];
   for (const annotation of selected) {
     const rubric = await loadRubricDefinition(

package/src/orchestrator/harness-registry.ts

@@ -0,0 +1,12 @@
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+import type { HarnessModelConfig, Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+
+export const registryWithHarness = (
+  registry: Registry,
+  config: Pick<ResolvedLuxConfig, "assertions">,
+  harness: HarnessModelConfig | undefined,
+): Registry =>
+  config.assertions?.some((assertion) => assertion.type === "rubric")
+    ? registry
+    : registry.withAssertion(createRubricAssertion(harness));

package/src/orchestrator/orchestrator.ts

@@ -1,4 +1,6 @@
 import { basename, extname, join, resolve } from "node:path";
+import { ContainerDriver } from "../container/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
 import { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type Answerer,
@@ -28,6 +30,7 @@ import {
   registryForEvalCase,
   valuesForFixtures,
 } from "./case-extensions.js";
+import { isBuiltinDriver } from "./config.js";
 import { validateDriverCapabilities } from "./driver-capabilities.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
@@ -249,6 +252,7 @@ export const resolveDriverReference = (
   return {
     name: driverNameOf(local),
     config: mergePluginConfig(driverConfigOf(base), driverConfigOf(local)),
+    execution: driverExecutionOf(local) ?? driverExecutionOf(base),
   };
 };
 
@@ -390,8 +394,20 @@ export class Orchestrator {
       `answerer '${answererName}'`,
     );
 
-    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires);
-    const driver = await driverPlugin.create(driverCfg);
+    const execution = driverExecutionOf(driverRef);
+    if (execution?.kind === "container" && request.experimentId)
+      throw new Error("container execution does not yet support controlled experiments");
+    if (
+      execution?.kind === "container" &&
+      !execution.driverModule &&
+      !isBuiltinDriver(driverPlugin)
+    )
+      throw new Error("custom container drivers require execution.driverModule in the image");
+    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires, execution);
+    const driver =
+      execution?.kind === "container"
+        ? new ContainerDriver(driverName, driverCfg, execution)
+        : await driverPlugin.create(driverCfg);
     let answerer: Answerer;
     try {
       answerer = await answererPlugin.create(answererCfg);
@@ -427,7 +443,7 @@ export class Orchestrator {
           permutationId: request.permutationId,
           answererName,
           answererConfig: answererCfg,
-          driverConfigHash: valueHash(driverCfg),
+          driverConfigHash: valueHash(execution ? { config: driverCfg, execution } : driverCfg),
           answererConfigHash: valueHash(answererCfg),
           validatedAssertionsHash,
           graderConfigHash: graderHash,