@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/compare.ts

@@ -2,6 +2,9 @@ import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  type ActionTiming,
+  type AgentActivity,
+  type DriverMetrics,
   emptyRunUsage,
   type GradingReport,
   GradingReportSchema,
@@ -14,8 +17,10 @@ import {
   stableJson,
   totalTokens,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { stableDriverIdentity } from "./comparison-identity.js";
 import { safeReadJson, walkFiles } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 
 export type ComparisonInput = {
   pathA: string;
@@ -35,9 +40,9 @@ export type GradingDelta = {
   id: string;
   type: string;
   name: string;
-  before: { passed: boolean; score: number } | null;
-  after: { passed: boolean; score: number } | null;
-  status: "added" | "removed" | "improved" | "regressed" | "unchanged";
+  before: { passed: boolean | null; score: number | null } | null;
+  after: { passed: boolean | null; score: number | null } | null;
+  status: "added" | "removed" | "improved" | "regressed" | "unchanged" | "unavailable";
 };
 
 export type RunComparison = {
@@ -48,17 +53,22 @@ export type RunComparison = {
     exitReason: { a: string; b: string };
     durationMsA: number | null;
     durationMsB: number | null;
+    actionTiming: { a: ActionTiming; b: ActionTiming };
+    driverMetrics: { a: DriverMetrics; b: DriverMetrics };
+    agentActivity: { a: AgentActivity[]; b: AgentActivity[] };
     interruptCounts: {
       a: { ask: number; approve: number; custom: number };
       b: { ask: number; approve: number; custom: number };
     };
     usage: { a: RunUsage; b: RunUsage };
+    agentUsageAvailable?: { a: boolean; b: boolean };
   } | null;
   grading: {
     casePassedA: boolean | null;
     casePassedB: boolean | null;
     caseScoreA: number | null;
     caseScoreB: number | null;
+    caseScoreComparable: boolean;
     deltas: GradingDelta[];
     improvements: number;
     regressions: number;
@@ -148,6 +158,9 @@ const assertionsByIdentity = (
   return out;
 };
 
+const measuredScore = (entry: GradingReport["assertions"][number]): number | null =>
+  entry.errorMessage !== undefined || entry.abstention !== undefined ? null : entry.score;
+
 const computeGradingDeltas = (
   a: GradingReport | null,
   b: GradingReport | null,
@@ -166,21 +179,24 @@ const computeGradingDeltas = (
     if (!ax && bx) status = "added";
     else if (ax && !bx) status = "removed";
     else if (ax && bx) {
-      if (ax.passed === bx.passed && ax.score === bx.score) status = "unchanged";
+      if (measuredScore(ax) === null || measuredScore(bx) === null) status = "unavailable";
+      else if (ax.passed === bx.passed && ax.score === bx.score) status = "unchanged";
       // A pass/fail flip dominates any score movement.
       else if (ax.passed !== bx.passed) status = bx.passed ? "improved" : "regressed";
       else status = ax.score < bx.score ? "improved" : "regressed";
     } else {
       status = "unchanged";
     }
-    if (status === "improved") improvements++;
-    if (status === "regressed") regressions++;
+    if (ax?.passed !== null && bx?.passed !== null) {
+      if (status === "improved") improvements++;
+      if (status === "regressed") regressions++;
+    }
     deltas.push({
       id: name,
       type: ax?.type ?? bx?.type ?? "unknown",
       name: ax?.id || bx?.id ? (ax?.name ?? bx?.name ?? name) : name,
-      before: ax ? { passed: ax.passed, score: ax.score } : null,
-      after: bx ? { passed: bx.passed, score: bx.score } : null,
+      before: ax ? { passed: ax.passed, score: measuredScore(ax) } : null,
+      after: bx ? { passed: bx.passed, score: measuredScore(bx) } : null,
       status,
     });
   }
@@ -226,6 +242,11 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
   if (stableJson(runtimeA) !== stableJson(runtimeB)) {
     caseMismatch.push({ field: "runtime", a: runtimeA, b: runtimeB });
   }
+  const environmentA = a.runJson?.provenance?.environment;
+  const environmentB = b.runJson?.provenance?.environment;
+  if (stableJson(environmentA) !== stableJson(environmentB)) {
+    caseMismatch.push({ field: "environment", a: environmentA, b: environmentB });
+  }
   const sourceA = a.runJson?.provenance?.source;
   const sourceB = b.runJson?.provenance?.source;
   const sourceHashUnavailable =
@@ -260,10 +281,26 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
           },
           durationMsA: durationFromMeta(a.runJson),
           durationMsB: durationFromMeta(b.runJson),
+          actionTiming: {
+            a: a.runJson?.actionTiming ?? {},
+            b: b.runJson?.actionTiming ?? {},
+          },
+          driverMetrics: {
+            a: a.runJson?.driverMetrics ?? {},
+            b: b.runJson?.driverMetrics ?? {},
+          },
+          agentActivity: {
+            a: a.runJson?.agentActivity ?? [],
+            b: b.runJson?.agentActivity ?? [],
+          },
           interruptCounts: {
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
           },
+          agentUsageAvailable: {
+            a: a.runJson !== null && hasAgentUsage(a.runJson),
+            b: b.runJson !== null && hasAgentUsage(b.runJson),
+          },
           usage: {
             a: a.runJson?.usage ?? emptyRunUsage(),
             b: b.runJson?.usage ?? emptyRunUsage(),
@@ -283,6 +320,9 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
       casePassedB: b.grading?.casePassed ?? null,
       caseScoreA: a.grading?.caseScore ?? null,
       caseScoreB: b.grading?.caseScore ?? null,
+      caseScoreComparable:
+        gradingScoreBasis(a.grading) !== null &&
+        gradingScoreBasis(a.grading) === gradingScoreBasis(b.grading),
       ...grading,
     },
     artifactsDelta: computeFileDelta(a.fileTree, b.fileTree),
@@ -296,7 +336,8 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
 export const hasRegression = (cmp: RunComparison): boolean =>
   cmp.caseMismatch.length > 0 ||
   cmp.grading.regressions > 0 ||
-  cmp.grading.deltas.some((delta) => delta.status === "removed") ||
+  cmp.grading.deltas.some((delta) => delta.after?.score === null && delta.after.passed !== null) ||
+  cmp.grading.deltas.some((delta) => delta.status === "removed" && delta.before?.passed !== null) ||
   (cmp.grading.casePassedA === true && cmp.grading.casePassedB !== true);
 
 const formatMismatchValue = (value: unknown): string => {
@@ -332,7 +373,11 @@ export const formatComparisonText = (cmp: RunComparison): string => {
     );
     const au = cmp.runDelta.usage.a;
     const bu = cmp.runDelta.usage.b;
-    lines.push(`  agent tokens: ${totalTokens(au.agent)} → ${totalTokens(bu.agent)}`);
+    const agentA =
+      cmp.runDelta.agentUsageAvailable?.a === false ? "unavailable" : totalTokens(au.agent);
+    const agentB =
+      cmp.runDelta.agentUsageAvailable?.b === false ? "unavailable" : totalTokens(bu.agent);
+    lines.push(`  agent tokens: ${agentA} → ${agentB}`);
     lines.push(
       `  harness tokens: ${totalTokens(harnessUsage(au))} → ${totalTokens(harnessUsage(bu))}`,
     );
@@ -341,13 +386,17 @@ export const formatComparisonText = (cmp: RunComparison): string => {
   lines.push("Grading:");
   lines.push(
     `  passed: ${String(cmp.grading.casePassedA)} → ${String(cmp.grading.casePassedB)}; ` +
-      `score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}`,
+      `score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}${cmp.grading.caseScoreComparable ? "" : " (change unavailable: scoring basis differs or is unknown)"}`,
   );
   lines.push(`  ${cmp.grading.improvements} improved, ${cmp.grading.regressions} regressed`);
   for (const d of cmp.grading.deltas) {
     if (d.status === "unchanged") continue;
-    const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓" }[d.status];
-    lines.push(`    ${arrow} ${d.name}`);
+    const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓", unavailable: "?" }[
+      d.status
+    ];
+    lines.push(
+      `    ${arrow} ${d.name}: ${d.before?.score ?? "unavailable"} → ${d.after?.score ?? "unavailable"}`,
+    );
   }
   lines.push("");
   lines.push("Artifacts:");

package/src/orchestrator/comparison-identity.ts

@@ -31,10 +31,12 @@ export const runComparisonIdentity = (metadata: RunMetadata): string =>
   valueHash({
     source: sourceIdentity(metadata),
     runtime: metadata.provenance?.runtime,
+    environment: metadata.provenance?.environment,
     caseHash: metadata.provenance?.caseHash,
     assertionsHash: metadata.provenance?.assertionsHash,
     fixturesHash: metadata.provenance?.fixturesHash,
     profileId: metadata.provenance?.profileId,
+    permutationId: metadata.provenance?.permutationId,
     driverConfigHash: metadata.provenance?.driverConfigHash,
     answererConfigHash: metadata.provenance?.answererConfigHash,
     evaluatorConfigHash: metadata.provenance?.evaluatorConfigHash,

package/src/orchestrator/config.ts

@@ -8,6 +8,7 @@ import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
 import { createRubricAssertion } from "../assertions/rubric/index.js";
+import { EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type AnswererPlugin,
   AnswererRefSchema,
@@ -17,13 +18,18 @@ import {
   type DriverPlugin,
   DriverRefSchema,
   formatZodIssues,
+  HarnessMatrixSchema,
   HarnessModelConfigSchema,
   type LifecycleHooksPlugin,
   lifecycleHooksPluginIssue,
   Registry,
 } from "../core/index.js";
+import { antigravityDriver } from "../drivers/antigravity/index.js";
 import { claudeCodeDriver } from "../drivers/claude-code/index.js";
 import { codexDriver } from "../drivers/codex/index.js";
+import { copilotDriver } from "../drivers/copilot/index.js";
+import { cursorDriver } from "../drivers/cursor/index.js";
+import { grokBuildDriver } from "../drivers/grok-build/index.js";
 import { mcpProbeDriver } from "../drivers/mcp/index.js";
 import { subprocessDriver } from "../drivers/subprocess/index.js";
 
@@ -34,6 +40,7 @@ const pluginLike = z.looseObject({
 
 export const ExperimentProfileSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     driver: DriverRefSchema.optional(),
     answerer: AnswererRefSchema.optional(),
     harness: HarnessModelConfigSchema.optional(),
@@ -67,8 +74,9 @@ const LifecycleHooksPluginSchema = z.custom<LifecycleHooksPlugin>().superRefine(
   if (issue) ctx.addIssue({ code: "custom", message: issue });
 });
 
-const LuxConfigSchema = z
+export const LuxConfigSchema = z
   .object({
+    environment: EnvironmentDeclarationSchema.optional(),
     casesRoot: z.string().optional(),
     fixturesRoot: z.string().optional(),
     runsRoot: z.string().default(".lux-runs"),
@@ -76,6 +84,10 @@ const LuxConfigSchema = z
     subjectRoot: z.string().optional(),
     defaultAnswerer: AnswererRefSchema.default("persona"),
     defaultDriver: DriverRefSchema.optional(),
+    /** Configuration inherited by every selected driver, then validated by that plugin. */
+    driverDefaults: z.record(z.string(), z.unknown()).optional(),
+    /** Driver, model, and effort permutations every case runs through by default. */
+    matrix: HarnessMatrixSchema.optional(),
     /** Shared provider/model defaults for persona answering and rubric judging. */
     harness: HarnessModelConfigSchema.optional(),
     concurrency: z.number().int().min(1).default(1),
@@ -162,8 +174,11 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
     ...(validated.fixturesRoot ? { fixturesRoot: resolve(root, validated.fixturesRoot) } : {}),
     defaultAnswerer: validated.defaultAnswerer,
     ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
+    ...(validated.driverDefaults ? { driverDefaults: validated.driverDefaults } : {}),
+    ...(validated.matrix ? { matrix: validated.matrix } : {}),
     ...(validated.harness ? { harness: validated.harness } : {}),
     concurrency: validated.concurrency,
+    ...(validated.environment ? { environment: validated.environment } : {}),
     ...(validated.profiles ? { profiles: validated.profiles } : {}),
     ...(validated.drivers ? { drivers: validated.drivers } : {}),
     ...(validated.answerers ? { answerers: validated.answerers } : {}),
@@ -175,9 +190,16 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
 const builtinDrivers: DriverPlugin<never>[] = [
   claudeCodeDriver,
   codexDriver,
+  cursorDriver,
+  copilotDriver,
+  grokBuildDriver,
+  antigravityDriver,
   subprocessDriver,
   mcpProbeDriver,
 ];
+export const isBuiltinDriver = (plugin: DriverPlugin): boolean =>
+  builtinDrivers.some((builtin) => builtin === plugin);
+
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,

package/src/orchestrator/discover.ts

@@ -13,6 +13,7 @@ export type DiscoveredCase = {
   relPath: string;
   /** `meta.tags` normalized to strings; empty when unset. */
   tags: string[];
+  cluster?: string;
 };
 
 /**
@@ -96,7 +97,13 @@ export const discoverCases = async (root: string): Promise<DiscoveryResult> => {
       problems.push({ path, reason: formatZodIssues(parsed.error) });
       continue;
     }
-    cases.push({ path, id: parsed.data.id ?? relPath, relPath, tags: tagsOf(parsed.data.meta) });
+    cases.push({
+      path,
+      id: parsed.data.id ?? relPath,
+      relPath,
+      tags: tagsOf(parsed.data.meta),
+      ...(parsed.data.meta?.cluster ? { cluster: parsed.data.meta.cluster } : {}),
+    });
   }
   return { cases, problems };
 };