@prismatic-io/lux 0.0.2-preview.36 → 0.0.2-preview.38

package/src/core/reference-defaults.ts

@@ -0,0 +1,47 @@
+import type { AnswererRef, DriverRef } from "./case.js";
+import { mergePluginConfig } from "./harness-matrix.js";
+
+export const withDriverDefaults = (
+  reference: DriverRef | undefined,
+  defaults: Record<string, unknown> | undefined,
+): DriverRef | undefined => {
+  if (!reference || !defaults) return reference;
+  const selected = typeof reference === "string" ? { name: reference } : reference;
+  return { ...selected, config: mergePluginConfig(defaults, selected.config) };
+};
+
+export const mergeDriverReference = (
+  base: DriverRef | undefined,
+  local: DriverRef | undefined,
+): DriverRef | undefined => {
+  if (local === undefined) return base;
+  if (base === undefined) return local;
+  const first = typeof base === "string" ? { name: base } : base;
+  const second = typeof local === "string" ? { name: local } : local;
+  if (first.name !== second.name) return local;
+  return {
+    name: second.name,
+    config: mergePluginConfig(first.config, second.config),
+    ...((second.execution ?? first.execution)
+      ? { execution: second.execution ?? first.execution }
+      : {}),
+  };
+};
+
+export const mergeAnswererReference = (
+  base: AnswererRef | undefined,
+  local: AnswererRef | undefined,
+): AnswererRef | undefined => {
+  const merged = mergeDriverReference(base, local);
+  if (typeof merged !== "object" || merged.name !== "persona") return merged;
+  const first = typeof base === "object" ? base.config : undefined;
+  const second = typeof local === "object" ? local.config : undefined;
+  if (!first || !second || typeof first !== "object" || typeof second !== "object") return merged;
+  if (!("provider" in second) || ("provider" in first && second.provider === first.provider))
+    return merged;
+  const config = { ...(merged.config as Record<string, unknown>) };
+  for (const key of ["model", "command", "reasoningEffort"]) {
+    if (!(key in second)) delete config[key];
+  }
+  return { name: merged.name, config };
+};

package/src/index.ts

@@ -4,7 +4,6 @@ export { DurationSchema, durationMs } from "./core/duration.js";
 // lifecycles. Explicit exports keep implementation and test helpers private.
 
 export { z } from "zod";
-
 // Bundled answerers and drivers. Test injection and protocol-normalization
 // helpers stay in their implementation modules.
 export {
@@ -140,6 +139,7 @@ export {
   type RubricConfig,
   RubricConfigSchema,
   type RubricEvidence,
+  type RubricEvidenceInput,
   RubricEvidenceSchema,
   RubricModelAccountingDetailSchema,
   rubricAssertion,
@@ -237,6 +237,7 @@ export {
   type PersistedCase,
   PersistedCaseSchema,
 } from "./core/case.js";
+export type { AssertionDefaults, EvalCaseDefaults } from "./core/case-defaults.js";
 export {
   CONVERSATION_TURN_REQUEST,
   type ConversationTurnRequest,

package/src/orchestrator/config.ts

@@ -84,6 +84,8 @@ export const LuxConfigSchema = z
     subjectRoot: z.string().optional(),
     defaultAnswerer: AnswererRefSchema.default("persona"),
     defaultDriver: DriverRefSchema.optional(),
+    /** Configuration inherited by every selected driver, then validated by that plugin. */
+    driverDefaults: z.record(z.string(), z.unknown()).optional(),
     /** Driver, model, and effort permutations every case runs through by default. */
     matrix: HarnessMatrixSchema.optional(),
     /** Shared provider/model defaults for persona answering and rubric judging. */
@@ -172,6 +174,7 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
     ...(validated.fixturesRoot ? { fixturesRoot: resolve(root, validated.fixturesRoot) } : {}),
     defaultAnswerer: validated.defaultAnswerer,
     ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
+    ...(validated.driverDefaults ? { driverDefaults: validated.driverDefaults } : {}),
     ...(validated.matrix ? { matrix: validated.matrix } : {}),
     ...(validated.harness ? { harness: validated.harness } : {}),
     concurrency: validated.concurrency,

package/src/orchestrator/doctor.ts

@@ -7,6 +7,7 @@ import { RubricConfigSchema } from "../assertions/rubric/index.js";
 import { driverExecutionOf } from "../core/driver-execution.js";
 import {
   casePermutations,
+  type DriverRef,
   type EvalCase,
   expandHarnessMatrix,
   type HarnessPermutation,
@@ -14,6 +15,7 @@ import {
   permutationId,
   type Registry,
 } from "../core/index.js";
+import { mergeDriverReference, withDriverDefaults } from "../core/reference-defaults.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
@@ -33,6 +35,8 @@ import {
   hasHarnessCatalogDrift,
 } from "./harness-catalog-drift.js";
 import { loadEvalCase } from "./loader.js";
+import { resolveAnswererReference } from "./orchestrator.js";
+import { resolveProfile } from "./profile.js";
 import { validatePluginConfig, validatePluginConfigAsync } from "./validate-plugin-config.js";
 
 export const DoctorDiagnosticSchema = z
@@ -120,42 +124,17 @@ const rawRefConfig = (reference: unknown): unknown =>
     ? (reference as { config?: unknown }).config
     : undefined;
 
-const mergeRecords = (base: unknown, local: unknown): unknown => {
-  if (
-    !base ||
-    typeof base !== "object" ||
-    Array.isArray(base) ||
-    !local ||
-    typeof local !== "object" ||
-    Array.isArray(local)
-  ) {
-    return local ?? base;
-  }
-  const merged: Record<string, unknown> = { ...(base as Record<string, unknown>) };
-  for (const [key, value] of Object.entries(local as Record<string, unknown>)) {
-    merged[key] = key in merged ? mergeRecords(merged[key], value) : value;
-  }
-  return merged;
-};
-
-const effectiveDriverReference = (local: unknown, fallback: unknown): unknown => {
-  if (!local) return fallback;
-  if (!fallback || refName(local) !== refName(fallback)) return local;
-  return {
-    name: refName(local),
-    config: mergeRecords(rawRefConfig(fallback), rawRefConfig(local)),
-    execution: driverExecutionOf(local) ?? driverExecutionOf(fallback),
-  };
-};
-
 const boundPermutation = (
   permutation: HarnessPermutation,
   config: ResolvedLuxConfig,
   registry: Registry,
-): unknown => {
+): DriverRef => {
   const plugin = registry.getDriver(permutation.driver);
   if (!plugin) throw new Error(`unknown driver '${permutation.driver}'`);
-  return permutationDriverRef(permutation, config.defaultDriver, plugin);
+  return withDriverDefaults(
+    permutationDriverRef(permutation, config.defaultDriver, plugin),
+    config.driverDefaults,
+  ) as DriverRef;
 };
 
 const caseDriverReferences = (
@@ -167,16 +146,20 @@ const caseDriverReferences = (
   const permutations = casePermutations(authored, config.matrix);
   if (permutations.length > 0) {
     return permutations.map((permutation) =>
-      effectiveDriverReference(local, boundPermutation(permutation, config, registry)),
+      mergeDriverReference(boundPermutation(permutation, config, registry), local),
     );
   }
   const fallbacks = [
-    config.defaultDriver,
-    ...Object.values(config.profiles ?? {}).map((profile) => profile.driver),
+    resolveProfile(config).driver,
+    ...Object.values(config.profiles ?? {}).map(
+      (profile) => resolveProfile(config, profile).driver,
+    ),
   ].filter((reference) => reference !== undefined);
   if (!local) return [...new Set(fallbacks)];
-  if (fallbacks.length === 0) return [local];
-  const references = fallbacks.map((fallback) => effectiveDriverReference(local, fallback));
+  if (fallbacks.length === 0) return [withDriverDefaults(local, config.driverDefaults)];
+  const references = fallbacks.map((fallback) =>
+    withDriverDefaults(mergeDriverReference(fallback, local), config.driverDefaults),
+  );
   return [
     ...new Map(references.map((reference) => [JSON.stringify(reference), reference])).values(),
   ];
@@ -210,7 +193,9 @@ const rubricCommands = (assertion: unknown, config: ResolvedLuxConfig): string[]
   const rubric = RubricConfigSchema.parse(assertion);
   const harnesses = [
     config.harness,
-    ...Object.values(config.profiles ?? {}).map((profile) => profile.harness ?? config.harness),
+    ...Object.values(config.profiles ?? {}).map(
+      (profile) => resolveProfile(config, profile).harness,
+    ),
   ];
   const commands = harnesses.map((harness) => {
     const provider = rubric.provider ?? harness?.provider ?? "claude-code";
@@ -227,7 +212,7 @@ const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): Cl
   const add = (label: string, command: string | undefined): void => {
     if (command) targets.push({ label, command });
   };
-  for (const command of driverCommands(config.defaultDriver, registry))
+  for (const command of driverCommands(resolveProfile(config).driver, registry))
     add("default driver", command);
   for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
     try {
@@ -250,11 +235,11 @@ const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): Cl
   for (const [id, profile] of Object.entries(config.profiles ?? {}).sort(([a], [b]) =>
     a.localeCompare(b),
   )) {
-    for (const command of driverCommands(profile.driver ?? config.defaultDriver, registry)) {
+    for (const command of driverCommands(resolveProfile(config, profile).driver, registry)) {
       add(`profile '${id}' driver`, command);
     }
-    add(`profile '${id}' answerer`, answererCommand(profile.answerer ?? config.defaultAnswerer));
-    const harness = profile.harness ?? config.harness;
+    add(`profile '${id}' answerer`, answererCommand(resolveProfile(config, profile).answerer));
+    const harness = resolveProfile(config, profile).harness;
     if (harness) {
       add(
         `profile '${id}' harness`,
@@ -356,18 +341,18 @@ const loadDoctorRegistry = (
 const configuredDriverReferences = (
   config: ResolvedLuxConfig,
 ): Array<readonly [string, unknown]> => [
-  ["default", config.defaultDriver],
+  ["default", resolveProfile(config).driver],
   ...Object.entries(config.profiles ?? {}).map(
-    ([id, profile]) => [`profile '${id}'`, profile.driver] as const,
+    ([id, profile]) => [`profile '${id}'`, resolveProfile(config, profile).driver] as const,
   ),
 ];
 
 const configuredAnswererReferences = (
   config: ResolvedLuxConfig,
 ): Array<readonly [string, unknown]> => [
-  ["default", config.defaultAnswerer],
+  ["default", resolveProfile(config).answerer],
   ...Object.entries(config.profiles ?? {}).map(
-    ([id, profile]) => [`profile '${id}'`, profile.answerer] as const,
+    ([id, profile]) => [`profile '${id}'`, resolveProfile(config, profile).answerer] as const,
   ),
 ];
 
@@ -651,7 +636,7 @@ const inspectCase = async (
     const authored = await loadEvalCase(discovered.path);
     const caseRegistry = registryForEvalCase(registry, authored);
     const drivers = caseDriverReferences(authored, config, caseRegistry);
-    const answerer = authored.answerer ?? config.defaultAnswerer;
+    const answerer = resolveAnswererReference(authored, config.defaultAnswerer, config.harness);
     validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder, authored.requires);
     validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [

package/src/orchestrator/experiment-context.ts

@@ -2,6 +2,7 @@ import type { Registry } from "../core/index.js";
 import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
+import { resolveProfile } from "./profile.js";
 
 export type CorpusSelection =
   | {
@@ -38,5 +39,5 @@ export const profileOrThrow = (config: ResolvedLuxConfig, profileId: string) =>
       `experiment profile '${profileId}' is not configured${available.length > 0 ? `; available: ${available.join(", ")}` : ""}`,
     );
   }
-  return profile;
+  return resolveProfile(config, profile);
 };

package/src/orchestrator/experiment-corpus.ts

@@ -131,7 +131,12 @@ export const agentCorpusHash = async (
           const profile = profileOrThrow(execution.config, profileId);
           const harness = profile.harness ?? execution.config.harness;
           const driver = bindSubjectRoot(
-            resolveDriverReference(bound, profile.driver ?? execution.config.defaultDriver),
+            resolveDriverReference(
+              bound,
+              profile.driver ?? execution.config.defaultDriver,
+              undefined,
+              execution.config.driverDefaults,
+            ),
             execution.loaded.subjectRoot,
           );
           const answerer = bindSubjectRoot(

package/src/orchestrator/experiment-evaluation.ts

@@ -512,6 +512,7 @@ export const evaluateAgentCandidate = async (options: {
   const defaultDriver = profile.driver ?? execution.config.defaultDriver;
   const registry = registryWithHarness(execution.registry, execution.config, harness);
   const orchestrator = new Orchestrator({
+    driverDefaults: execution.config.driverDefaults,
     environment: profile.environment ?? execution.config.environment,
     registry,
     defaultAnswerer: profile.answerer ?? execution.config.defaultAnswerer,

package/src/orchestrator/experiment-identity.ts

@@ -132,7 +132,12 @@ const trustedExecutableEntrypoints = async (
       for (const profileId of execution.loaded.campaign.profiles) {
         const profile = profileOrThrow(execution.config, profileId);
         const driver = bindSubjectRoot(
-          resolveDriverReference(authored, profile.driver ?? execution.config.defaultDriver),
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
           execution.loaded.subjectRoot,
         );
         for (const command of referencedDriverRuntime(driver, execution.registry).commands)
@@ -198,7 +203,12 @@ export const runtimeCliReferences = async (
         const profile = profileOrThrow(execution.config, profileId);
         const harness = profile.harness ?? execution.config.harness;
         const driver = bindSubjectRoot(
-          resolveDriverReference(authored, profile.driver ?? execution.config.defaultDriver),
+          resolveDriverReference(
+            authored,
+            profile.driver ?? execution.config.defaultDriver,
+            undefined,
+            execution.config.driverDefaults,
+          ),
           execution.loaded.subjectRoot,
         );
         const answerer = bindSubjectRoot(
@@ -275,7 +285,12 @@ export const validateAgentCaseProfileDrivers = async (
     for (const profileId of execution.loaded.campaign.profiles) {
       const profile = profileOrThrow(execution.config, profileId);
       const reference = bindSubjectRoot(
-        resolveDriverReference(authored, profile.driver ?? execution.config.defaultDriver),
+        resolveDriverReference(
+          authored,
+          profile.driver ?? execution.config.defaultDriver,
+          undefined,
+          execution.config.driverDefaults,
+        ),
         execution.loaded.subjectRoot,
       );
       const name = typeof reference === "string" ? reference : reference.name;

package/src/orchestrator/grading.ts

@@ -273,10 +273,9 @@ export const gradeRun = async (
         type: a.type,
         name,
         passed: a.informational ? null : passed,
-        ...(a.informational ? { informational: true } : {}),
+        ...(a.informational ? { informational: true } : { threshold }),
         score: result.score,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         ...(result.comment !== undefined ? { comment: result.comment } : {}),
         detail: result.detail,
@@ -291,10 +290,9 @@ export const gradeRun = async (
         type: a.type,
         name,
         passed: a.informational ? null : false,
-        ...(a.informational ? { informational: true } : {}),
+        ...(a.informational ? { informational: true } : { threshold }),
         score: 0,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         errorMessage: message,
       };

package/src/orchestrator/orchestrator.ts

@@ -17,13 +17,17 @@ import {
   type HarnessModelConfig,
   type LifecycleHooksPlugin,
   mergeHarnessModelConfig,
-  mergePluginConfig,
   type Registry,
   type RunMetadata,
   type RunObserver,
   type SuiteLifecycleHookContext,
   valueHash,
 } from "../core/index.js";
+import {
+  mergeAnswererReference,
+  mergeDriverReference,
+  withDriverDefaults,
+} from "../core/reference-defaults.js";
 import {
   fixturesForEvalCase,
   mergeLifecycleFixtures,
@@ -116,6 +120,7 @@ export type OrchestratorOptions = {
   defaultAnswerer?: AnswererRef;
   /** Driver used when a case names none. */
   defaultDriver?: DriverRef;
+  driverDefaults?: Record<string, unknown> | undefined;
   /** Shared provider/model defaults for built-in harness components. */
   harness?: HarnessModelConfig;
   /** Versioned project resource fixtures. */
@@ -233,6 +238,7 @@ export const resolveDriverReference = (
   evalCase: EvalCase,
   fallback: DriverRef | undefined,
   permutation?: DriverRef | undefined,
+  defaults?: Record<string, unknown> | undefined,
 ): DriverRef => {
   const local = evalCase.driver;
   if (permutation && local && driverNameOf(local) !== driverNameOf(permutation)) {
@@ -240,20 +246,12 @@ export const resolveDriverReference = (
       `case is pinned to driver '${driverNameOf(local)}' but the run selected '${driverNameOf(permutation)}'`,
     );
   }
-  const base = permutation ?? fallback;
-  if (!local) {
-    if (!base) throw new Error("case names no driver and no defaultDriver is configured");
-    return base;
-  }
-  if (!base || driverNameOf(local) !== driverNameOf(base)) return local;
-
-  // Naming the same driver opts into project defaults. A case object only
-  // needs to carry the fields it wants to override (for example a timeout).
-  return {
-    name: driverNameOf(local),
-    config: mergePluginConfig(driverConfigOf(base), driverConfigOf(local)),
-    execution: driverExecutionOf(local) ?? driverExecutionOf(base),
-  };
+  const reference = withDriverDefaults(
+    mergeDriverReference(permutation ?? fallback, local),
+    defaults,
+  );
+  if (!reference) throw new Error("case names no driver and no defaultDriver is configured");
+  return reference;
 };
 
 export const resolveAnswererReference = (
@@ -262,7 +260,18 @@ export const resolveAnswererReference = (
   harness: HarnessModelConfig | undefined,
   override?: string,
 ): AnswererRef => {
-  const ref = override ?? evalCase.answerer ?? fallback;
+  const fallbackName = typeof fallback === "string" ? fallback : fallback.name;
+  const inherited =
+    harness && fallbackName === "persona"
+      ? {
+          name: "persona",
+          config: mergeHarnessModelConfig(
+            harness,
+            typeof fallback === "object" ? fallback.config : undefined,
+          ),
+        }
+      : fallback;
+  const ref = mergeAnswererReference(inherited, override ?? evalCase.answerer) as AnswererRef;
   const name = typeof ref === "string" ? ref : ref.name;
   if (!harness || name !== "persona") return ref;
   return {
@@ -310,6 +319,7 @@ export class Orchestrator {
   readonly registry: Registry;
   private readonly defaultAnswerer: AnswererRef;
   private readonly defaultDriver: DriverRef | undefined;
+  private readonly driverDefaults: Record<string, unknown> | undefined;
   private readonly harness: HarnessModelConfig | undefined;
   private readonly lifecycleFixtures: readonly LifecycleHooksPlugin[];
 
@@ -320,6 +330,7 @@ export class Orchestrator {
     this.registry = options.registry;
     this.defaultAnswerer = options.defaultAnswerer ?? "persona";
     this.defaultDriver = options.defaultDriver;
+    this.driverDefaults = options.driverDefaults;
     this.harness = options.harness;
     this.lifecycleFixtures = [...(options.lifecycleFixtures ?? [])];
   }
@@ -359,7 +370,7 @@ export class Orchestrator {
     );
 
     const driverRef = bindSubjectRoot(
-      resolveDriverReference(evalCase, this.defaultDriver, request.driver),
+      resolveDriverReference(evalCase, this.defaultDriver, request.driver, this.driverDefaults),
       request.subjectRoot,
     );
     const driverName = driverNameOf(driverRef);

package/src/orchestrator/profile.ts

@@ -0,0 +1,24 @@
+import {
+  mergeAnswererReference,
+  mergeDriverReference,
+  withDriverDefaults,
+} from "../core/reference-defaults.js";
+import type { ExperimentProfile, ResolvedLuxConfig } from "./config.js";
+
+/** Used by ordinary runs, doctor, and experiment execution and identity alike. */
+export const resolveProfile = (
+  config: Pick<
+    ResolvedLuxConfig,
+    "defaultDriver" | "defaultAnswerer" | "harness" | "environment" | "driverDefaults"
+  >,
+  profile: ExperimentProfile = {},
+): ExperimentProfile => ({
+  ...profile,
+  driver: withDriverDefaults(
+    mergeDriverReference(config.defaultDriver, profile.driver),
+    config.driverDefaults,
+  ),
+  answerer: mergeAnswererReference(config.defaultAnswerer, profile.answerer),
+  harness: profile.harness ?? config.harness,
+  environment: profile.environment ?? config.environment,
+});

package/viewer/detail.js

@@ -96,10 +96,21 @@ const eventTime = (event, startedAt) => {
   return `+${duration(Math.max(0, event.ts - start))}`;
 };
 
+const assertionScoreLabel = (assertion) => {
+  const label = `${assertion.name} scored ${percentage(assertionScore(assertion))}`;
+  if (assertion.informational || assertion.passed === null) return `${label} (informational)`;
+  if (typeof assertion.threshold === "number") {
+    return `${label}; threshold ${percentage(assertion.threshold)}`;
+  }
+  return label;
+};
+
 const assertionMeta = (assertion) =>
   [
     assertion.type,
-    `threshold ${finite(assertion.threshold)}`,
+    !assertion.informational && assertion.passed !== null && typeof assertion.threshold === "number"
+      ? `threshold ${finite(assertion.threshold)}`
+      : null,
     `weight ${finite(assertion.weight)}`,
     duration(assertion.durationMs),
     assertion.usage ? `${compactNumber(totalTokens(assertion.usage))} judge tokens` : null,
@@ -334,10 +345,10 @@ export const renderRunDetail = (item, detail) => {
                         ${scoreBar(
                           assertionScore(assertion),
                           assertionPresentation(assertion).tone,
-                          `${assertion.name} scored ${percentage(
-                            assertionScore(assertion),
-                          )}${assertion.passed === null ? " (informational)" : `; threshold ${percentage(assertion.threshold)}`}`,
-                          assertion.passed === null ? null : assertion.threshold,
+                          assertionScoreLabel(assertion),
+                          assertion.informational || assertion.passed === null
+                            ? null
+                            : assertion.threshold,
                         )}
                         ${assertionDiagnostics(assertion)}
                       </span>