@prismatic-io/lux 0.0.2-preview.27 → 0.0.2-preview.28

package/src/orchestrator/doctor.ts

@@ -4,6 +4,7 @@ import { resolve } from "node:path";
 import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
 import {
   casePermutations,
   type EvalCase,
@@ -143,6 +144,7 @@ const effectiveDriverReference = (local: unknown, fallback: unknown): unknown =>
   return {
     name: refName(local),
     config: mergeRecords(rawRefConfig(fallback), rawRefConfig(local)),
+    execution: driverExecutionOf(local) ?? driverExecutionOf(fallback),
   };
 };
 
@@ -511,7 +513,7 @@ const caseDriverFailure = (
       rawRefConfig(reference),
       `case '${caseId}' driver '${name}'`,
     );
-    validateDriverCapabilities(plugin, config, required);
+    validateDriverCapabilities(plugin, config, required, driverExecutionOf(reference));
     return null;
   } catch (error) {
     return firstLine(error);

package/src/orchestrator/driver-capabilities.ts

@@ -1,12 +1,17 @@
 import type { DriverCapability, DriverPlugin } from "../core/driver.js";
+import type { DriverExecution } from "../core/driver-execution.js";
 
 export const validateDriverCapabilities = (
   plugin: DriverPlugin,
   config: unknown,
   required: readonly DriverCapability[] = [],
+  execution?: DriverExecution,
 ): void => {
   if (required.length === 0) return;
-  const capabilities = plugin.runtime?.(config).capabilities;
+  const capabilities = {
+    ...plugin.runtime?.(config).capabilities,
+    ...(execution?.kind === "container" ? { isolation: true } : {}),
+  };
   const missing = required.filter((capability) => capabilities?.[capability] !== true);
   if (missing.length > 0) {
     throw new Error(

package/src/orchestrator/driver-identity.ts

@@ -1,4 +1,5 @@
 import type { DriverPlugin, DriverRuntime } from "../core/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
 import type { Registry } from "../core/registry.js";
 import { cliIdentity } from "./experiment-runtime-identity.js";
 import { validatePluginConfig } from "./validate-plugin-config.js";
@@ -21,7 +22,14 @@ const resolvedDriver = (
 
 export const referencedDriverRuntime = (reference: unknown, registry: Registry): DriverRuntime => {
   const resolved = resolvedDriver(reference, registry);
-  return resolved?.plugin.runtime?.(resolved.config) ?? { commands: [], capabilities: {} };
+  const runtime = resolved?.plugin.runtime?.(resolved.config) ?? { commands: [], capabilities: {} };
+  const execution = driverExecutionOf(reference);
+  return execution?.kind === "container"
+    ? {
+        commands: [execution.command ?? "docker"],
+        capabilities: { ...runtime.capabilities, isolation: true },
+      }
+    : runtime;
 };
 
 export const experimentDriverIdentity = async (
@@ -29,6 +37,8 @@ export const experimentDriverIdentity = async (
   registry: Registry,
   rootDir: string,
 ): Promise<unknown> => {
+  if (driverExecutionOf(reference)?.kind === "container")
+    throw new Error("container execution does not yet support controlled experiments");
   const resolved = resolvedDriver(reference, registry);
   if (!resolved) return null;
   const { plugin, config } = resolved;

package/src/orchestrator/orchestrator.ts

@@ -1,4 +1,6 @@
 import { basename, extname, join, resolve } from "node:path";
+import { ContainerDriver } from "../container/driver.js";
+import { driverExecutionOf } from "../core/driver-execution.js";
 import { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type Answerer,
@@ -28,6 +30,7 @@ import {
   registryForEvalCase,
   valuesForFixtures,
 } from "./case-extensions.js";
+import { isBuiltinDriver } from "./config.js";
 import { validateDriverCapabilities } from "./driver-capabilities.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
@@ -249,6 +252,7 @@ export const resolveDriverReference = (
   return {
     name: driverNameOf(local),
     config: mergePluginConfig(driverConfigOf(base), driverConfigOf(local)),
+    execution: driverExecutionOf(local) ?? driverExecutionOf(base),
   };
 };
 
@@ -390,8 +394,20 @@ export class Orchestrator {
       `answerer '${answererName}'`,
     );
 
-    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires);
-    const driver = await driverPlugin.create(driverCfg);
+    const execution = driverExecutionOf(driverRef);
+    if (execution?.kind === "container" && request.experimentId)
+      throw new Error("container execution does not yet support controlled experiments");
+    if (
+      execution?.kind === "container" &&
+      !execution.driverModule &&
+      !isBuiltinDriver(driverPlugin)
+    )
+      throw new Error("custom container drivers require execution.driverModule in the image");
+    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires, execution);
+    const driver =
+      execution?.kind === "container"
+        ? new ContainerDriver(driverName, driverCfg, execution)
+        : await driverPlugin.create(driverCfg);
     let answerer: Answerer;
     try {
       answerer = await answererPlugin.create(answererCfg);
@@ -427,7 +443,7 @@ export class Orchestrator {
           permutationId: request.permutationId,
           answererName,
           answererConfig: answererCfg,
-          driverConfigHash: valueHash(driverCfg),
+          driverConfigHash: valueHash(execution ? { config: driverCfg, execution } : driverCfg),
           answererConfigHash: valueHash(answererCfg),
           validatedAssertionsHash,
           graderConfigHash: graderHash,