@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/run-workspace.ts

@@ -0,0 +1,62 @@
+import { cp, lstat, mkdir, mkdtemp, realpath, rename, rm, rmdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import type { Artifact } from "../core/index.js";
+
+/** Keep package-manager discovery and repository ignore rules outside subject workspaces. */
+export class RunWorkspace {
+  private archived = false;
+  readonly path: string;
+
+  private constructor(path: string) {
+    this.path = path;
+  }
+
+  static async create(): Promise<RunWorkspace> {
+    const root = join(tmpdir(), `lux-workspaces-${process.getuid?.() ?? "user"}`);
+    await mkdir(root, { recursive: true, mode: 0o700 });
+    const metadata = await lstat(root);
+    if (
+      !metadata.isDirectory() ||
+      metadata.isSymbolicLink() ||
+      (process.getuid && metadata.uid !== process.getuid())
+    ) {
+      throw new Error(
+        `Lux workspace root must be a real directory owned by the current user: ${root}`,
+      );
+    }
+    return new RunWorkspace(await realpath(await mkdtemp(join(root, "run-"))));
+  }
+
+  async archive(destination: string, artifacts: Artifact[] = []): Promise<Artifact[]> {
+    if (!this.archived) {
+      const metadata = await lstat(this.path);
+      if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
+        throw new Error(`run workspace must remain a real directory: ${this.path}`);
+      }
+      await rmdir(destination).catch((error: NodeJS.ErrnoException) => {
+        if (error.code !== "ENOENT") throw error;
+      });
+      try {
+        await rename(this.path, destination);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "EXDEV") throw error;
+        await cp(this.path, destination, { recursive: true, verbatimSymlinks: true });
+        await rm(this.path, { recursive: true });
+      }
+      this.archived = true;
+    }
+    return artifacts.map((artifact) => {
+      if (!artifact.root) return artifact;
+      const path = relative(this.path, resolve(artifact.root));
+      if (path === "") {
+        const { root: _root, ...captured } = artifact;
+        return captured;
+      }
+      if (path === ".." || path.startsWith(`..${sep}`) || isAbsolute(path)) {
+        return artifact;
+      }
+      return { ...artifact, root: join(destination, path) };
+    });
+  }
+}

package/src/orchestrator/suite-compare.ts

@@ -5,6 +5,7 @@ import {
   MetricSummarySchema,
   type SuiteSample,
   type SuiteSummary,
+  suiteGroupKey,
   summarizeNumbers,
 } from "./suite-summary.js";
 
@@ -18,6 +19,7 @@ const ComparedOutcomeSchema = z
 export const SuiteCaseComparisonSchema = z
   .object({
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     before: ComparedOutcomeSchema.nullable(),
     after: ComparedOutcomeSchema.nullable(),
     passRateDelta: z.number().nullable(),
@@ -40,6 +42,7 @@ export const SuiteComparisonSchema = z
       z
         .object({
           canonicalCaseId: z.string(),
+          permutationId: z.exactOptional(z.string()),
           repetition: z.number().int().positive(),
           before: z.string().nullable(),
           after: z.string().nullable(),
@@ -54,9 +57,10 @@ export type SuiteComparison = z.infer<typeof SuiteComparisonSchema>;
 const samplesByCase = (summary: SuiteSummary): Map<string, SuiteSample[]> => {
   const out = new Map<string, SuiteSample[]>();
   for (const sample of summary.samples) {
-    const entries = out.get(sample.canonicalCaseId) ?? [];
+    const key = suiteGroupKey(sample);
+    const entries = out.get(key) ?? [];
     entries.push(sample);
-    out.set(sample.canonicalCaseId, entries);
+    out.set(key, entries);
   }
   return out;
 };
@@ -70,10 +74,19 @@ const pairedMetricDeltas = (
     const a = before[index];
     const b = after[index];
     if (!a || !b) continue;
-    const names = new Set([...Object.keys(a.metrics), ...Object.keys(b.metrics)]);
+    const am: Record<string, number> = {
+      ...a.metrics,
+      ...(a.score !== undefined ? { score: a.score } : {}),
+    };
+    const bm: Record<string, number> = {
+      ...b.metrics,
+      ...(b.score !== undefined ? { score: b.score } : {}),
+    };
+    const names = new Set([...Object.keys(am), ...Object.keys(bm)]);
     for (const name of names) {
-      const av = a.metrics[name];
-      const bv = b.metrics[name];
+      if (name === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis)) continue;
+      const av = am[name];
+      const bv = bm[name];
       if (av === undefined || bv === undefined) continue;
       const values = deltas.get(name) ?? [];
       values.push(bv - av);
@@ -95,14 +108,23 @@ export const compareSuiteSummaries = (
   options: { passRateTolerance?: number } = {},
 ): SuiteComparison => {
   const passRateTolerance = options.passRateTolerance ?? 0;
-  const aCases = new Map(a.cases.map((entry) => [entry.canonicalCaseId, entry]));
-  const bCases = new Map(b.cases.map((entry) => [entry.canonicalCaseId, entry]));
+  const aCases = new Map(a.cases.map((entry) => [suiteGroupKey(entry), entry]));
+  const bCases = new Map(b.cases.map((entry) => [suiteGroupKey(entry), entry]));
   const aSamples = samplesByCase(a);
   const bSamples = samplesByCase(b);
-  const ids = new Set([...aCases.keys(), ...bCases.keys()]);
-  const identityMismatches = [...ids].sort().flatMap((canonicalCaseId) => {
-    const before = aSamples.get(canonicalCaseId) ?? [];
-    const after = bSamples.get(canonicalCaseId) ?? [];
+  const groups = new Map(
+    [...a.cases, ...b.cases].map((entry) => [
+      suiteGroupKey(entry),
+      {
+        canonicalCaseId: entry.canonicalCaseId,
+        ...(entry.permutationId ? { permutationId: entry.permutationId } : {}),
+      },
+    ]),
+  );
+  const keys = [...groups.keys()].sort();
+  const identityMismatches = keys.flatMap((key) => {
+    const before = aSamples.get(key) ?? [];
+    const after = bSamples.get(key) ?? [];
     return Array.from({ length: Math.max(before.length, after.length) }, (_, index) => {
       const beforeHash = before[index]?.identityHash ?? null;
       const afterHash = after[index]?.identityHash ?? null;
@@ -110,7 +132,7 @@ export const compareSuiteSummaries = (
         ? []
         : [
             {
-              canonicalCaseId,
+              ...groups.get(key),
               repetition: index + 1,
               before: beforeHash,
               after: afterHash,
@@ -118,9 +140,9 @@ export const compareSuiteSummaries = (
           ];
     }).flat();
   });
-  const cases = [...ids].sort().map((canonicalCaseId) => {
-    const before = aCases.get(canonicalCaseId);
-    const after = bCases.get(canonicalCaseId);
+  const cases = keys.map((key) => {
+    const before = aCases.get(key);
+    const after = bCases.get(key);
     let passRateDelta: number | null = null;
     if (before && after) {
       passRateDelta = after.passRate - before.passRate;
@@ -128,16 +150,13 @@ export const compareSuiteSummaries = (
       passRateDelta = -before.passRate;
     }
     return {
-      canonicalCaseId,
+      ...groups.get(key),
       before: before ? { attempts: before.attempts, passRate: before.passRate } : null,
       after: after ? { attempts: after.attempts, passRate: after.passRate } : null,
       passRateDelta,
       regressed:
         Boolean(before) && (!after || after.passRate + passRateTolerance < (before?.passRate ?? 0)),
-      pairedMetricDeltas: pairedMetricDeltas(
-        aSamples.get(canonicalCaseId) ?? [],
-        bSamples.get(canonicalCaseId) ?? [],
-      ),
+      pairedMetricDeltas: pairedMetricDeltas(aSamples.get(key) ?? [], bSamples.get(key) ?? []),
     };
   });
   return SuiteComparisonSchema.parse({
@@ -182,13 +201,18 @@ export const formatSuiteComparisonText = (comparison: SuiteComparison): string =
   if (comparison.identityMismatches.length > 0) {
     lines.push(`  identity mismatches: ${comparison.identityMismatches.length}`);
     for (const mismatch of comparison.identityMismatches) {
-      lines.push(`    ! ${mismatch.canonicalCaseId} repetition ${mismatch.repetition}`);
+      lines.push(`    ! ${suiteGroupKey(mismatch)} repetition ${mismatch.repetition}`);
     }
   }
   for (const entry of comparison.cases) {
-    if (!entry.regressed) continue;
+    if (entry.regressed) {
+      lines.push(
+        `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      );
+    }
+    const score = entry.pairedMetricDeltas.score;
     lines.push(
-      `    ↓ ${entry.canonicalCaseId}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      `    ${suiteGroupKey(entry)} score change: ${score ? `${score.mean >= 0 ? "+" : ""}${score.mean} (${score.n} paired)` : "unavailable"}`,
     );
   }
   return lines.join("\n");

package/src/orchestrator/suite-runs.ts

@@ -10,6 +10,7 @@ const SUITE_FILE = "suite.json";
 const SuiteItemSchema = z.object({
   caseId: z.string(),
   canonicalCaseId: z.string(),
+  permutationId: z.exactOptional(z.string()),
   status: z.enum(["pending", "running", "completed"]),
   runDir: z.string().nullable(),
   exitReason: z.union([z.literal("done"), z.string()]).nullable(),
@@ -64,7 +65,7 @@ export type OpenedSuiteRun = {
 export const openSuiteRun = async (options: {
   runsRoot: string;
   concurrency: number;
-  items: { id: string; caseId?: string | undefined }[];
+  items: { id: string; caseId?: string | undefined; permutationId?: string | undefined }[];
   startedAt?: Date | undefined;
   profileId?: string | undefined;
   experimentId?: string | undefined;
@@ -110,6 +111,7 @@ export const openSuiteRun = async (options: {
     items: options.items.map((item) => ({
       caseId: item.id,
       canonicalCaseId: item.caseId ?? item.id,
+      ...(item.permutationId ? { permutationId: item.permutationId } : {}),
       status: "pending",
       runDir: null,
       exitReason: null,
@@ -126,6 +128,7 @@ export const loadSuiteRun = async (path: string): Promise<SuiteRun | null> =>
 
 export const listSuiteRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<SuiteRun & { suiteDir: string }>> => {
   const root = join(runsRoot, SUITES_DIR);
   if (!existsSync(root)) return [];
@@ -133,7 +136,15 @@ export const listSuiteRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const manifests = await Promise.all(
-    dirs.map(async (suiteDir) => ({ suiteDir, suite: await loadSuiteRun(suiteDir) })),
+    dirs.map(async (suiteDir) => {
+      try {
+        return { suiteDir, suite: await loadSuiteRun(suiteDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(suiteDir, error);
+        return { suiteDir, suite: null };
+      }
+    }),
   );
   return manifests
     .flatMap(({ suiteDir, suite }) => (suite ? [{ ...suite, suiteDir }] : []))

package/src/orchestrator/suite-summary.ts

@@ -1,9 +1,17 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import {
+  type DriverMetric,
+  DriverMetricUnitSchema,
+  harnessUsage,
+  isUsageEmpty,
+  type RunMetadata,
+} from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
 import type { SuiteRun } from "./suite-runs.js";
 
@@ -16,6 +24,9 @@ export const MetricSummarySchema = z.object({
   mean: z.number(),
   median: z.number(),
   p90: z.number(),
+  unit: z.exactOptional(DriverMetricUnitSchema),
+  label: z.exactOptional(z.string()),
+  better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 
@@ -32,18 +43,30 @@ export type OutcomeSummary = z.infer<typeof OutcomeSummarySchema>;
 
 const CaseSummarySchema = OutcomeSummarySchema.extend({
   canonicalCaseId: z.string(),
+  permutationId: z.exactOptional(z.string()),
   metrics: z.record(z.string(), MetricSummarySchema),
 });
+
+/** Repetitions pool per case and harness permutation, never across permutations. */
+export const suiteGroupKey = (entry: {
+  canonicalCaseId: string;
+  permutationId?: string | undefined;
+}): string =>
+  entry.permutationId === undefined
+    ? entry.canonicalCaseId
+    : `${entry.canonicalCaseId} [${entry.permutationId}]`;
 export type CaseSummary = z.infer<typeof CaseSummarySchema>;
 
 export const SuiteSampleSchema = z.object({
   index: z.number().int().nonnegative(),
   caseId: z.string(),
   canonicalCaseId: z.string(),
+  permutationId: z.exactOptional(z.string()),
   exitReason: z.string().nullable(),
   passed: z.boolean().nullable(),
   identityHash: z.string().nullable(),
   score: z.exactOptional(z.number()),
+  scoreBasis: z.exactOptional(z.string().nullable()),
   metrics: z.record(z.string(), z.number()),
 });
 export type SuiteSample = z.infer<typeof SuiteSampleSchema>;
@@ -59,13 +82,22 @@ export const SuiteSummarySchema = z.object({
 });
 export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
 
-type MetricSamples = Map<string, number[]>;
+type MetricSamples = Map<
+  string,
+  { values: number[]; descriptor?: Omit<DriverMetric, "value"> | undefined }
+>;
 
-const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+const add = (
+  samples: MetricSamples,
+  name: string,
+  value: number | undefined,
+  descriptor?: Omit<DriverMetric, "value">,
+): void => {
   if (value === undefined || !Number.isFinite(value)) return;
-  const values = samples.get(name) ?? [];
-  values.push(value);
-  samples.set(name, values);
+  const sample = samples.get(name) ?? { values: [], descriptor };
+  if (!sample.descriptor && descriptor) sample.descriptor = descriptor;
+  sample.values.push(value);
+  samples.set(name, sample);
 };
 
 const cacheHitRate = (usage: {
@@ -84,7 +116,7 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
     add(samples, "durationMs", Math.max(0, ended - started));
   }
 
-  if (!isUsageEmpty(metadata.usage.agent)) {
+  if (hasAgentUsage(metadata)) {
     add(samples, "agent.input", metadata.usage.agent.input);
     add(samples, "agent.output", metadata.usage.agent.output);
     add(samples, "agent.cacheRead", metadata.usage.agent.cacheRead);
@@ -137,6 +169,78 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
   add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
   add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
   add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+  add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+  add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+  add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+  const subagents = (metadata.agentActivity ?? []).filter(
+    (activity) => activity.agent.role === "subagent",
+  );
+  add(samples, "subagents.count", subagents.length, {
+    unit: "count",
+    label: "Subagents spawned",
+    better: "neutral",
+  });
+  add(
+    samples,
+    "subagents.events",
+    subagents.reduce((sum, activity) => sum + activity.eventCount, 0),
+    { unit: "count", label: "Subagent events", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.total, 0),
+    { unit: "count", label: "Subagent tool calls", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls.failed",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.failed, 0),
+    { unit: "count", label: "Failed subagent tool calls", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.tokens",
+    subagents.reduce((sum, activity) => sum + activity.usage.total, 0),
+    { unit: "tokens", label: "Subagent tokens", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.actions",
+    subagents.reduce((sum, activity) => sum + activity.actions.total, 0),
+    { unit: "count", label: "Subagent actions", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.mutations",
+    subagents.reduce((sum, activity) => sum + activity.actions.mutations, 0),
+    { unit: "count", label: "Subagent mutations", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.totalDurationMs",
+    subagents.reduce((sum, activity) => sum + activity.durationMs, 0),
+    { unit: "milliseconds", label: "Total subagent time", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.maxDurationMs",
+    subagents.length > 0 ? Math.max(...subagents.map((activity) => activity.durationMs)) : 0,
+    { unit: "milliseconds", label: "Longest subagent", better: "lower" },
+  );
+  const firstSubagentAction = subagents
+    .map((activity) => activity.actions.timeToFirstActionMs)
+    .filter((value): value is number => value !== undefined)
+    .sort((a, b) => a - b)[0];
+  add(samples, "subagents.timeToFirstActionMs", firstSubagentAction, {
+    unit: "milliseconds",
+    label: "First subagent action",
+    better: "lower",
+  });
+  for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+    const { value, ...descriptor } = metric;
+    add(samples, `driver.${name}`, value, descriptor);
+  }
 };
 
 const percentile = (sorted: readonly number[], fraction: number): number => {
@@ -165,7 +269,10 @@ const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary>
   Object.fromEntries(
     [...samples.entries()]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([name, values]) => [name, summarizeNumbers(values)]),
+      .map(([name, { values, descriptor }]) => [
+        name,
+        MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+      ]),
   );
 
 const summarizeOutcomes = (
@@ -214,14 +321,15 @@ export const buildSuiteSummary = (
 
   const byCase = new Map<string, typeof entries>();
   for (const entry of entries) {
-    const group = byCase.get(entry.item.canonicalCaseId) ?? [];
+    const key = suiteGroupKey(entry.item);
+    const group = byCase.get(key) ?? [];
     group.push(entry);
-    byCase.set(entry.item.canonicalCaseId, group);
+    byCase.set(key, group);
   }
 
   const cases = [...byCase.entries()]
     .sort(([a], [b]) => a.localeCompare(b))
-    .map(([canonicalCaseId, caseEntries]) => {
+    .map(([, caseEntries]) => {
       const metrics: MetricSamples = new Map();
       for (const entry of caseEntries) {
         const result = entry.result?.result;
@@ -229,8 +337,10 @@ export const buildSuiteSummary = (
         add(metrics, "score", result.grading.caseScore ?? undefined);
         collectMetadata(metrics, result.metadata);
       }
+      const first = caseEntries[0]?.item;
       return {
-        canonicalCaseId,
+        canonicalCaseId: first?.canonicalCaseId ?? "",
+        ...(first?.permutationId ? { permutationId: first.permutationId } : {}),
         ...summarizeOutcomes(caseEntries),
         metrics: summarizeMetrics(metrics),
       };
@@ -246,19 +356,23 @@ export const buildSuiteSummary = (
     samples: entries.map((entry, index) => {
       const samples: MetricSamples = new Map();
       const result = entry.result?.result;
-      if (result) collectMetadata(samples, result.metadata);
+      if (result) {
+        add(samples, "score", result.grading.caseScore ?? undefined);
+        collectMetadata(samples, result.metadata);
+      }
       return {
         index,
         caseId: entry.item.caseId,
         canonicalCaseId: entry.item.canonicalCaseId,
+        ...(entry.item.permutationId ? { permutationId: entry.item.permutationId } : {}),
         exitReason: entry.result?.exitReason ?? null,
         passed: entry.result?.casePassed ?? null,
         identityHash: result ? runComparisonIdentity(result.metadata) : null,
         ...(result?.grading.caseScore !== null && result?.grading.caseScore !== undefined
-          ? { score: result.grading.caseScore }
+          ? { score: result.grading.caseScore, scoreBasis: gradingScoreBasis(result.grading) }
           : {}),
         metrics: Object.fromEntries(
-          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+          [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),
         ),
       };
     }),