@prismatic-io/lux 0.0.2-preview.14 → 0.0.2-preview.16

package/src/orchestrator/orchestrator.ts

@@ -1,4 +1,5 @@
-import { basename, extname, isAbsolute, join, resolve } from "node:path";
+import { basename, extname, join, resolve } from "node:path";
+import { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "../core/environment.js";
 import {
   type Answerer,
   type AnswererModelCallAccounting,
@@ -24,6 +25,7 @@ import {
   registryForEvalCase,
   valuesForFixtures,
 } from "./case-extensions.js";
+import { validateDriverCapabilities } from "./driver-capabilities.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
   assertionEvaluatorIdentities,
@@ -102,6 +104,7 @@ const recordLifecycleCleanupFailure = async (result: RunResult, error: unknown):
 };
 
 export type OrchestratorOptions = {
+  environment?: EnvironmentDeclaration | undefined;
   registry: Registry;
   /** Answerer used when a case names none. Defaults to "persona". */
   defaultAnswerer?: AnswererRef;
@@ -114,6 +117,8 @@ export type OrchestratorOptions = {
 };
 
 export type RunRequest = {
+  /** Effective parent suite worker limit; direct runs default to one. */
+  suiteConcurrency?: number | undefined;
   /** Path to a TS case file. Mutually exclusive with `case`. */
   casePath?: string;
   /** Pre-loaded EvalCase. Mutually exclusive with `casePath`. */
@@ -265,191 +270,22 @@ const driverConfigOf = (ref: DriverRef): unknown =>
 const answererConfigOf = (ref: AnswererRef): unknown =>
   typeof ref === "object" ? ref.config : undefined;
 
-const EXPERIMENT_BEHAVIOR_SOURCE_ARGS = new Set([
-  "--continue",
-  "-c",
-  "--fork-session",
-  "--resume",
-  "-r",
-  "--add-dir",
-  "--append-system-prompt-file",
-  "--mcp-config",
-  "--plugin-dir",
-  "--settings",
-  "--setting-sources",
-  "--system-prompt-file",
-]);
-
-const CODEX_EXPERIMENT_SOURCE_ARGS = new Set([
-  "--add-dir",
-  "--cd",
-  "-C",
-  "--config",
-  "-c",
-  "--profile",
-  "-p",
-]);
-
-const bindSubprocessExperimentConfig = (config: Record<string, unknown>): unknown => {
-  if (typeof config.command !== "string" || !isAbsolute(config.command)) {
-    throw new Error(
-      "experiment subprocess commands must be absolute direct wrappers so Lux can fingerprint the invoked behavior",
-    );
-  }
-  if (typeof config.cwd === "string") {
-    throw new Error(
-      "experiment runs do not allow an explicit subprocess cwd; use the candidate subject and staged fixtures",
-    );
-  }
-  if (Array.isArray(config.args) && config.args.length > 0) {
-    throw new Error(
-      "experiment subprocesses do not allow args; put stable behavior in the fingerprinted direct wrapper",
-    );
-  }
-  if (isRecord(config.env) && Object.keys(config.env).length > 0) {
-    throw new Error(
-      "experiment subprocesses do not allow custom env; put stable behavior in the fingerprinted direct wrapper",
-    );
-  }
-  return { ...config, inheritEnv: false };
-};
-
-type AgentDriverName = "claude-code" | "codex";
-
-const validateAgentExperimentConfig = (
-  driverName: AgentDriverName,
-  config: Record<string, unknown>,
-): { driverEnv: Record<string, unknown>; extraArgs: string[] } => {
-  const driverEnv = isRecord(config.env) ? config.env : {};
-  const commandResolutionOverride = Object.keys(driverEnv).find((name) =>
-    ["PATH", "PATHEXT"].includes(name.toUpperCase()),
-  );
-  if (commandResolutionOverride) {
-    throw new Error(
-      `experiment runs do not allow driver env.${commandResolutionOverride}; command resolution must match the fingerprinted host executable`,
-    );
-  }
-  if (typeof config.cwd === "string") {
-    throw new Error(
-      `experiment runs do not allow an explicit ${driverName} cwd; use the candidate subject and staged fixtures`,
-    );
-  }
-
-  const extraArgs = Array.isArray(config.extraArgs)
-    ? config.extraArgs.filter((value): value is string => typeof value === "string")
-    : [];
-  const disallowedArgs =
-    driverName === "claude-code" ? EXPERIMENT_BEHAVIOR_SOURCE_ARGS : CODEX_EXPERIMENT_SOURCE_ARGS;
-  const behaviorSourceArg = extraArgs.find((arg) =>
-    [...disallowedArgs].some((name) => arg === name || arg.startsWith(`${name}=`)),
-  );
-  if (behaviorSourceArg) {
-    throw new Error(
-      `experiment runs do not allow behavior-bearing extraArgs '${behaviorSourceArg}'; use structured driver fields so Lux can fingerprint the source`,
-    );
-  }
-
-  const isolation = isRecord(config.isolation) ? config.isolation : {};
-  const structuredPaths = [
-    ...(driverName === "claude-code" && Array.isArray(config.pluginDirs) ? config.pluginDirs : []),
-    ...(Array.isArray(config.addDirs) ? config.addDirs : []),
-    ...(driverName === "claude-code" && Array.isArray(isolation.allowRead)
-      ? isolation.allowRead
-      : []),
-  ].filter((value): value is string => typeof value === "string");
-  const relativeSource = structuredPaths.find((path) => !isAbsolute(path));
-  if (relativeSource) {
-    throw new Error(
-      `experiment driver source paths must be absolute after subjectPath binding: ${relativeSource}`,
-    );
-  }
-  return { driverEnv, extraArgs };
-};
-
-const bindCodexExperimentConfig = (
-  config: Record<string, unknown>,
-  extraArgs: readonly string[],
-): unknown => {
-  if (typeof config.profile === "string") {
-    throw new Error(
-      "experiment runs do not allow a Codex profile; use inline driver config so Lux can fingerprint it",
-    );
-  }
-  if (config.ephemeral === false) {
-    throw new Error("experiment runs require ephemeral Codex sessions");
-  }
-  const inlineConfig = isRecord(config.config) ? config.config : {};
-  const externalConfigKey = ((): string | undefined => {
-    const visit = (value: Record<string, unknown>, parent = ""): string | undefined => {
-      for (const [key, nested] of Object.entries(value)) {
-        const path = parent ? `${parent}.${key}` : key;
-        if (/(?:mcp_servers|instructions_file|notify|project_doc)/i.test(key)) return path;
-        if (isRecord(nested)) {
-          const found = visit(nested, path);
-          if (found) return found;
-        }
-      }
-      return undefined;
-    };
-    return visit(inlineConfig);
-  })();
-  if (externalConfigKey) {
-    throw new Error(
-      `experiment runs do not allow path-bearing Codex config '${externalConfigKey}'`,
-    );
-  }
-  const appServer = config.interactionMode !== "exec";
-  return {
-    ...config,
-    ephemeral: true,
-    ignoreUserConfig: true,
-    ignoreRules: true,
-    ...(appServer && !config.isolation ? { isolation: { allowRead: [] } } : {}),
-    extraArgs,
-  };
-};
-
-const bindClaudeExperimentConfig = (
-  config: Record<string, unknown>,
-  extraArgs: readonly string[],
-  driverEnv: Record<string, unknown>,
-): unknown => {
-  if (config.isolation) return config;
-  return {
-    ...config,
-    extraArgs: [
-      ...extraArgs,
-      "--setting-sources",
-      "user",
-      "--strict-mcp-config",
-      "--settings",
-      JSON.stringify({ disableAllHooks: true }),
-    ],
-    env: {
-      ...driverEnv,
-      CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",
-      CLAUDE_CODE_DISABLE_CLAUDE_MDS: "1",
-      CLAUDE_CODE_SKIP_PROMPT_HISTORY: "1",
-      CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1",
-      CLAUDE_CODE_AUTO_CONNECT_IDE: "false",
-      CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL: "1",
-      DISABLE_AUTOUPDATER: "1",
-    },
-  };
-};
-
 export const boundExperimentDriverConfig = (
-  driverName: string,
+  plugin: import("../core/driver.js").DriverPlugin,
   config: unknown,
   experimentId: string | undefined,
 ): unknown => {
-  if (!experimentId || !isRecord(config)) return config;
-  if (driverName === "subprocess") return bindSubprocessExperimentConfig(config);
-  if (driverName !== "claude-code" && driverName !== "codex") return config;
-  const { driverEnv, extraArgs } = validateAgentExperimentConfig(driverName, config);
-  return driverName === "codex"
-    ? bindCodexExperimentConfig(config, extraArgs)
-    : bindClaudeExperimentConfig(config, extraArgs, driverEnv);
+  if (!experimentId) return config;
+  if (!plugin.experiment) {
+    throw new Error(
+      `driver '${plugin.name}' supports ordinary eval runs but does not declare controlled experiment support`,
+    );
+  }
+  return validatePluginConfig(
+    plugin.configSchema,
+    plugin.experiment.prepare(config),
+    `driver '${plugin.name}' experiment configuration`,
+  );
 };
 
 const closeQuietly = async (close: () => Promise<void>, label: string): Promise<void> => {
@@ -463,6 +299,7 @@ const closeQuietly = async (close: () => Promise<void>, label: string): Promise<
 };
 
 export class Orchestrator {
+  private readonly environment: EnvironmentDeclaration | undefined;
   private evaluatorHash: Promise<string> | undefined;
   readonly registry: Registry;
   private readonly defaultAnswerer: AnswererRef;
@@ -471,6 +308,9 @@ export class Orchestrator {
   private readonly lifecycleFixtures: readonly LifecycleHooksPlugin[];
 
   constructor(options: OrchestratorOptions) {
+    this.environment = options.environment
+      ? EnvironmentDeclarationSchema.parse(options.environment)
+      : undefined;
     this.registry = options.registry;
     this.defaultAnswerer = options.defaultAnswerer ?? "persona";
     this.defaultDriver = options.defaultDriver;
@@ -534,7 +374,7 @@ export class Orchestrator {
     // Validate both configs before instantiating either plugin so a
     // schema failure can't leak a half-initialized driver/answerer.
     const driverCfg = boundExperimentDriverConfig(
-      driverName,
+      driverPlugin,
       validatePluginConfig(
         driverPlugin.configSchema,
         driverConfigOf(driverRef),
@@ -548,6 +388,7 @@ export class Orchestrator {
       `answerer '${answererName}'`,
     );
 
+    validateDriverCapabilities(driverPlugin, driverCfg, evalCase.requires);
     const driver = await driverPlugin.create(driverCfg);
     let answerer: Answerer;
     try {
@@ -599,6 +440,7 @@ export class Orchestrator {
             skipJudge: request.skipJudge ?? false,
           }),
           fixtureValues: valuesForFixtures(lifecycleFixtures),
+          environment: this.environment,
           driver,
           answerer,
         },
@@ -797,6 +639,7 @@ export class Orchestrator {
             observer: handle ?? observer,
             abortSignal: opts.abortSignal,
             suiteId: suite.manifest.suiteId,
+            suiteConcurrency: concurrency,
             subjectRoot: opts.subjectRoot,
             subjectHash: opts.subjectHash,
             profileId: opts.profileId,

package/src/orchestrator/paired-effects.ts

@@ -0,0 +1,209 @@
+import { z } from "zod";
+import { clusterKey } from "../core/evaluation-clusters.js";
+import {
+  bootstrapMeanConfidenceInterval,
+  type CandidateEvaluation,
+  type ConfidenceConfig,
+  ConfidenceIntervalSchema,
+} from "../core/index.js";
+
+type CaseSamples = NonNullable<CandidateEvaluation["caseSamples"]>;
+type Evaluation = CandidateEvaluation & { repetitions: number };
+
+export const clusterCaseSamples = (
+  groups: readonly (CaseSamples | undefined)[],
+): CaseSamples | undefined => {
+  if (groups.some((group) => group === undefined)) return undefined;
+  const result = new Map<string, CaseSamples[string]>();
+  for (const group of groups) {
+    for (const [id, sample] of Object.entries(group ?? {})) {
+      const current = result.get(id);
+      if (!current) {
+        result.set(id, structuredClone(sample));
+        continue;
+      }
+      if (current.identityHash !== sample.identityHash) current.identityHash = null;
+      if (current.clusterId !== sample.clusterId) current.identityHash = null;
+      const metrics = new Map(Object.entries(current.metrics));
+      for (const [metric, value] of Object.entries(sample.metrics)) {
+        const before = metrics.get(metric);
+        metrics.set(
+          metric,
+          before
+            ? {
+                mean: (before.mean * before.n + value.mean * value.n) / (before.n + value.n),
+                n: before.n + value.n,
+              }
+            : { ...value },
+        );
+      }
+      current.metrics = Object.fromEntries(metrics);
+    }
+  }
+  return Object.fromEntries(result);
+};
+
+const caseGroups = (samples: CaseSamples): Map<string, string[]> => {
+  const groups = new Map<string, string[]>();
+  for (const id of Object.keys(samples).sort()) {
+    const key = clusterKey(id, samples[id]?.clusterId);
+    groups.set(key, [...(groups.get(key) ?? []), id]);
+  }
+  return groups;
+};
+
+export const groupedCaseMetricSamples = (
+  samples: CaseSamples | undefined,
+  repetitions: number,
+): Record<string, number[]> | undefined => {
+  if (!samples || !Object.values(samples).some((sample) => sample.clusterId !== undefined))
+    return undefined;
+  const groups = caseGroups(samples);
+  const metrics = new Set(Object.values(samples).flatMap((sample) => Object.keys(sample.metrics)));
+  return Object.fromEntries(
+    [...metrics].flatMap((metric) => {
+      if (
+        Object.values(samples).some(
+          (sample) =>
+            !Object.hasOwn(sample.metrics, metric) || sample.metrics[metric]?.n !== repetitions,
+        )
+      )
+        return [];
+      return [
+        [
+          metric,
+          [...groups.values()].map(
+            (ids) =>
+              ids.reduce((sum, id) => sum + (samples[id]?.metrics[metric]?.mean ?? 0), 0) /
+              ids.length,
+          ),
+        ],
+      ];
+    }),
+  );
+};
+
+export const PairedEffectSchema = z.object({
+  caseIds: z.array(z.string()),
+  n: z.number().int().nonnegative(),
+  caseCount: z.number().int().nonnegative().optional(),
+  clusters: z.record(z.string(), z.array(z.string())).optional(),
+  meanDelta: z.number().finite().nullable(),
+  interval: ConfidenceIntervalSchema.nullable(),
+  note: z.string().nullable(),
+});
+export type PairedEffect = z.infer<typeof PairedEffectSchema>;
+
+export const confidenceGateIssue = (
+  interval: z.infer<typeof ConfidenceIntervalSchema>,
+): string | undefined =>
+  interval.method === "percentile-bootstrap" && interval.lower === interval.upper
+    ? "zero-width percentile-bootstrap interval cannot establish promotion confidence; use Hoeffding for bounded means or collect more varied independent evidence"
+    : undefined;
+
+const pairedDeltas = (
+  before: Evaluation | undefined,
+  after: Evaluation | undefined,
+  metric: string,
+): { caseIds: string[]; values: number[]; clusters: Record<string, string[]> } => {
+  if (!before?.caseSamples || !after?.caseSamples)
+    throw new Error("keyed case evidence unavailable");
+  if (
+    before.profileId !== after.profileId ||
+    before.split !== after.split ||
+    before.repetitions !== after.repetitions
+  ) {
+    throw new Error("paired evidence requires matching profile, split, and repetitions");
+  }
+  const caseIds = Object.keys(before.caseSamples).sort();
+  const afterIds = Object.keys(after.caseSamples).sort();
+  const beforeCases = Object.keys(before.casePassRates).sort();
+  const afterCases = Object.keys(after.casePassRates).sort();
+  if (
+    caseIds.length === 0 ||
+    [afterIds, beforeCases, afterCases].some(
+      (ids) => JSON.stringify(ids) !== JSON.stringify(caseIds),
+    )
+  ) {
+    throw new Error("paired evidence requires identical canonical case sets");
+  }
+  const values = caseIds.map((id) => {
+    const a = before.caseSamples?.[id];
+    const b = after.caseSamples?.[id];
+    if (!a?.identityHash || a.identityHash !== b?.identityHash)
+      throw new Error(`paired case '${id}' has incompatible evaluation identity`);
+    if (a.clusterId !== b.clusterId)
+      throw new Error(`paired case '${id}' has incompatible cluster membership`);
+    const av = Object.hasOwn(a.metrics, metric) ? a.metrics[metric] : undefined;
+    const bv = Object.hasOwn(b.metrics, metric) ? b.metrics[metric] : undefined;
+    if (!av || !bv || av.n !== before.repetitions || bv.n !== after.repetitions) {
+      throw new Error(
+        `paired metric '${metric}' lacks complete repetition coverage for case '${id}'`,
+      );
+    }
+    return bv.mean - av.mean;
+  });
+  const byCase = new Map(caseIds.map((id, index) => [id, values[index] ?? 0]));
+  const groups = caseGroups(before.caseSamples);
+  return {
+    caseIds,
+    clusters: Object.fromEntries(groups),
+    values: [...groups.values()].map(
+      (ids) => ids.reduce((sum, id) => sum + (byCase.get(id) ?? 0), 0) / ids.length,
+    ),
+  };
+};
+
+export const pairedEffect = (
+  before: Evaluation | undefined,
+  after: Evaluation | undefined,
+  metric: string,
+  confidence?: ConfidenceConfig,
+): PairedEffect => {
+  let paired: ReturnType<typeof pairedDeltas>;
+  try {
+    paired = pairedDeltas(before, after, metric);
+  } catch (error) {
+    return {
+      caseIds: [],
+      n: 0,
+      caseCount: 0,
+      clusters: {},
+      meanDelta: null,
+      interval: null,
+      note: error instanceof Error ? error.message : String(error),
+    };
+  }
+  const result: PairedEffect = {
+    caseIds: paired.caseIds,
+    n: paired.values.length,
+    caseCount: paired.caseIds.length,
+    clusters: paired.clusters,
+    meanDelta: paired.values.reduce((sum, value) => sum + value, 0) / paired.values.length,
+    interval: null,
+    note: null,
+  };
+  if (!confidence) return result;
+  try {
+    if (confidence.method === "wilson")
+      throw new Error("Wilson cannot estimate paired differences");
+    if (confidence.method === "hoeffding") {
+      const interval = bootstrapMeanConfidenceInterval(
+        paired.values.map((value) => (value + 1) / 2),
+        confidence,
+      );
+      result.interval = {
+        ...interval,
+        estimate: 2 * interval.estimate - 1,
+        lower: 2 * interval.lower - 1,
+        upper: 2 * interval.upper - 1,
+      };
+    } else {
+      result.interval = bootstrapMeanConfidenceInterval(paired.values, confidence);
+    }
+    result.note = confidenceGateIssue(result.interval) ?? null;
+  } catch (error) {
+    result.note = error instanceof Error ? error.message : String(error);
+  }
+  return result;
+};

package/src/orchestrator/provenance.ts

@@ -1,6 +1,8 @@
 import { existsSync } from "node:fs";
 import { lstat, readFile, readlink } from "node:fs/promises";
+import { availableParallelism, cpus, release, totalmem } from "node:os";
 import { dirname, join, parse, relative, resolve, sep } from "node:path";
+import type { EnvironmentDeclaration } from "../core/environment.js";
 import type { RunProvenance } from "../core/index.js";
 import { contentHash, exec, valueHash } from "../core/index.js";
 
@@ -113,13 +115,30 @@ export const runtimeIdentity = (): RunProvenance["runtime"] => ({
   arch: process.arch,
   node: process.version,
   ...(process.versions.bun ? { bun: process.versions.bun } : {}),
+  host: {
+    osRelease: release(),
+    cpuModels: [...new Set(cpus().map((cpu) => cpu.model))].sort(),
+    logicalCpuCount: cpus().length,
+    availableParallelism: availableParallelism(),
+    totalMemoryBytes: totalmem(),
+  },
 });
 
-export const runProvenance = async (casePath: string | null): Promise<RunProvenance> => {
+export const runProvenance = async (
+  casePath: string | null,
+  options: {
+    environment?: EnvironmentDeclaration | undefined;
+    concurrency?: number | undefined;
+  } = {},
+): Promise<RunProvenance> => {
   const runtime = runtimeIdentity();
-  if (!casePath) return { runtime };
+  const environment = {
+    ...(options.environment ? { declared: options.environment } : {}),
+    concurrency: options.concurrency ?? 1,
+  };
+  if (!casePath) return { runtime, environment };
   const root = gitRootAbove(casePath);
-  if (!root) return { runtime, source: { casePath: resolve(casePath) } };
+  if (!root) return { runtime, environment, source: { casePath: resolve(casePath) } };
   let git = repositories.get(root);
   if (!git) {
     git = inspectRepository(root);
@@ -128,6 +147,7 @@ export const runProvenance = async (casePath: string | null): Promise<RunProvena
   const state = await git;
   return {
     runtime,
+    environment,
     source: {
       casePath: portablePath(relative(root, resolve(casePath))),
       ...(state ? { git: state } : {}),

package/src/orchestrator/run-execution.ts

@@ -1,6 +1,7 @@
 import { cp } from "node:fs/promises";
 import { dirname, isAbsolute, resolve } from "node:path";
 import { z } from "zod";
+import type { EnvironmentDeclaration } from "../core/environment.js";
 import {
   type AgentDriver,
   type Answerer,
@@ -60,6 +61,7 @@ export const INLINE_CASE_PATH = "<inline>";
 type RunDirCarrier = { luxRunDir?: string };
 type InterruptCounts = { ask: number; approve: number; custom: number };
 export type RunWithPluginsArgs = {
+  environment?: EnvironmentDeclaration | undefined;
   request: RunRequest;
   casePath: string;
   authoredCase: EvalCase;
@@ -173,8 +175,17 @@ const SummaryModelAccountingSchema = z.object({
   unpricedModelCalls: z.number().int().nonnegative().optional(),
 });
 
-const usageFromSummary = (summary: unknown): TokenUsage | null =>
-  SummaryUsageSchema.safeParse(summary).data?.usage ?? null;
+const usageFromSummary = (summary: unknown): TokenUsage | null => {
+  if (!summary || typeof summary !== "object") return null;
+  const raw = (summary as { usage?: unknown }).usage;
+  if (!raw || typeof raw !== "object") return null;
+  const counts = raw as Record<string, unknown>;
+  const hasCount = ["input", "output", "cacheRead", "cacheCreation"].some((key) => {
+    const value = counts[key];
+    return typeof value === "number" && Number.isInteger(value) && value >= 0;
+  });
+  return hasCount ? (SummaryUsageSchema.safeParse(summary).data?.usage ?? null) : null;
+};
 
 const costFromSummary = (summary: unknown): number | undefined => {
   if (!summary || typeof summary !== "object") return undefined;
@@ -320,6 +331,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const startedAt = args.request.startedAt ?? new Date();
     const provenance = await runProvenance(
       args.casePath === INLINE_CASE_PATH ? null : args.casePath,
+      { environment: args.environment, concurrency: args.request.suiteConcurrency },
     );
     const observer = guardRunObserver(args.request.observer);
     const runDir = await openRunDir({
@@ -528,6 +540,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       exitReason: "error",
       driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
+      agentUsageAvailability: "unavailable",
       usage: {
         agent: emptyUsage(),
         answerer: readAnswererUsage(this.args.answerer),
@@ -580,8 +593,9 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
+    const agentUsage = usageFromSummary(summary);
     const usage: RunUsage = {
-      agent: usageFromSummary(summary) ?? emptyUsage(),
+      agent: agentUsage ?? emptyUsage(),
       answerer: readAnswererUsage(this.args.answerer),
       grading: emptyUsage(),
     };
@@ -597,6 +611,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage,
+      agentUsageAvailability: agentUsage === null ? "unavailable" : "reported",
       modelAccounting: {
         agentCalls,
         answererCalls: answererAccounting.calls,

package/src/orchestrator/run-lifecycle.ts

@@ -23,6 +23,7 @@ type RunLifecycleContext = {
   phase: RunFailurePhase;
   result: unknown;
   error: unknown;
+  failed: boolean;
 };
 type RunLifecycleInput = { effects: ErasedEffects };
 type RunLifecycleOutcome =
@@ -56,7 +57,7 @@ const runLifecycleMachine = setup({
   actions: {
     captureError: assign(({ context, event }) => {
       const error = "error" in event ? event.error : new Error("run lifecycle failed");
-      return { error, phase: isRunTraceLimit(error) ? "trace" : context.phase };
+      return { error, failed: true, phase: isRunTraceLimit(error) ? "trace" : context.phase };
     }),
     captureResult: assign({
       result: ({ event }) => ("output" in event ? event.output : undefined),
@@ -73,6 +74,7 @@ const runLifecycleMachine = setup({
     phase: "setup",
     result: undefined,
     error: undefined,
+    failed: false,
   }),
   initial: "preparing",
   states: {
@@ -153,7 +155,7 @@ const runLifecycleMachine = setup({
     failed: { type: "final" },
   },
   output: ({ context }) => {
-    if (context.error !== undefined) {
+    if (context.failed) {
       return { status: "failed", error: context.error };
     }
     return { status: "succeeded", result: context.result };

package/src/orchestrator/suite-runs.ts

@@ -126,6 +126,7 @@ export const loadSuiteRun = async (path: string): Promise<SuiteRun | null> =>
 
 export const listSuiteRuns = async (
   runsRoot: string,
+  options: { onReadError?: (path: string, error: unknown) => void } = {},
 ): Promise<Array<SuiteRun & { suiteDir: string }>> => {
   const root = join(runsRoot, SUITES_DIR);
   if (!existsSync(root)) return [];
@@ -133,7 +134,15 @@ export const listSuiteRuns = async (
     .filter((entry) => entry.isDirectory())
     .map((entry) => join(root, entry.name));
   const manifests = await Promise.all(
-    dirs.map(async (suiteDir) => ({ suiteDir, suite: await loadSuiteRun(suiteDir) })),
+    dirs.map(async (suiteDir) => {
+      try {
+        return { suiteDir, suite: await loadSuiteRun(suiteDir) };
+      } catch (error) {
+        if (!options.onReadError) throw error;
+        options.onReadError(suiteDir, error);
+        return { suiteDir, suite: null };
+      }
+    }),
   );
   return manifests
     .flatMap(({ suiteDir, suite }) => (suite ? [{ ...suite, suiteDir }] : []))

package/src/orchestrator/suite-summary.ts

@@ -8,6 +8,7 @@ import {
   isUsageEmpty,
   type RunMetadata,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
@@ -102,7 +103,7 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
     add(samples, "durationMs", Math.max(0, ended - started));
   }
 
-  if (!isUsageEmpty(metadata.usage.agent)) {
+  if (hasAgentUsage(metadata)) {
     add(samples, "agent.input", metadata.usage.agent.input);
     add(samples, "agent.output", metadata.usage.agent.output);
     add(samples, "agent.cacheRead", metadata.usage.agent.cacheRead);