@prismatic-io/lux 0.0.1 → 0.0.2-preview.10

package/src/drivers/claude-code/parse-events.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import {
+  type AgentAttribution,
   type Interrupt,
   type ParsedDriverEvent,
   parseAgentUsage,
@@ -8,6 +9,39 @@ import {
   toolResultProgress,
 } from "../../core/index.js";
 
+const claudeAgent = (parentToolUseId?: string): AgentAttribution =>
+  parentToolUseId
+    ? {
+        id: `claude-code:${parentToolUseId}`,
+        role: "subagent",
+        parentId: "claude-code",
+        spawnToolUseId: parentToolUseId,
+      }
+    : { id: "claude-code", role: "main" };
+
+const isSubagentTool = (name: string): boolean => name === "Agent" || name === "Task";
+
+const spawnedClaudeAgent = (
+  toolUseId: string,
+  input: unknown,
+  parentToolUseId?: string,
+): AgentAttribution => {
+  const details =
+    typeof input === "object" && input !== null ? (input as Record<string, unknown>) : {};
+  const name = [details.name, details.subagent_type, details.agent, details.role].find(
+    (value): value is string => typeof value === "string" && value.length > 0,
+  );
+  const model = typeof details.model === "string" && details.model ? details.model : undefined;
+  return {
+    id: `claude-code:${toolUseId}`,
+    role: "subagent",
+    parentId: claudeAgent(parentToolUseId).id,
+    spawnToolUseId: toolUseId,
+    ...(name ? { name } : {}),
+    ...(model ? { model } : {}),
+  };
+};
+
 const ContentBlockSchema = z.discriminatedUnion("type", [
   z.object({ type: z.literal("text"), text: z.string() }),
   z.object({ type: z.literal("tool_use"), id: z.string(), name: z.string(), input: z.unknown() }),
@@ -88,6 +122,7 @@ type PendingTurn = {
   asked: boolean;
   thought: boolean;
   text?: string;
+  agent: AgentAttribution;
 };
 
 type AssistantContent = {
@@ -96,6 +131,7 @@ type AssistantContent = {
   asked: boolean;
   thought: boolean;
   firstText?: string;
+  agent: AgentAttribution;
 };
 
 const questionTextOf = (raw: unknown): string =>
@@ -225,6 +261,30 @@ class ClaudeEventParser {
       if (!block) continue;
       events.push(block);
       if (block.tag !== "progress") continue;
+      const resultPayload = block.progress.payload as {
+        name?: unknown;
+        toolUseId?: unknown;
+        isError?: unknown;
+      };
+      if (
+        typeof resultPayload.name === "string" &&
+        isSubagentTool(resultPayload.name) &&
+        typeof resultPayload.toolUseId === "string"
+      ) {
+        events.push({
+          tag: "progress",
+          progress: {
+            id: `agent-completed-${resultPayload.toolUseId}`,
+            kind: "agent-completed",
+            payload: {
+              agentId: `claude-code:${resultPayload.toolUseId}`,
+              status: resultPayload.isError === true ? "failed" : "completed",
+            },
+            agent: claudeAgent(),
+            ts: block.progress.ts,
+          },
+        });
+      }
       const text = (block.progress.payload as { text?: unknown }).text;
       const failure = typeof text === "string" ? toolInfrastructureFailure(text) : null;
       if (failure) {
@@ -244,20 +304,21 @@ class ClaudeEventParser {
 
     const toolUseId = block.tool_use_id;
     const content = block.content ?? [];
+    const progress = toolResultProgress(
+      this.eventSequence++,
+      {
+        name: this.toolNames.get(toolUseId) ?? "unknown",
+        toolUseId,
+        content,
+        isError: block.is_error === true,
+        text: toolResultText(content),
+        ...(parentToolUseId ? { parentToolUseId } : {}),
+      },
+      Date.now(),
+    );
     return {
       tag: "progress",
-      progress: toolResultProgress(
-        this.eventSequence++,
-        {
-          name: this.toolNames.get(toolUseId) ?? "unknown",
-          toolUseId,
-          content,
-          isError: block.is_error === true,
-          text: toolResultText(content),
-          ...(parentToolUseId ? { parentToolUseId } : {}),
-        },
-        Date.now(),
-      ),
+      progress: { ...progress, agent: claudeAgent(parentToolUseId) },
     };
   }
 
@@ -284,6 +345,7 @@ class ClaudeEventParser {
       tools: [],
       asked: false,
       thought: false,
+      agent: claudeAgent(event.parent_tool_use_id),
     };
     for (const rawBlock of event.message.content) {
       const block = ContentBlockSchema.safeParse(rawBlock).data;
@@ -323,6 +385,7 @@ class ClaudeEventParser {
         kind: "agent-message",
         payload: { text, ...(parentToolUseId ? { parentToolUseId } : {}) },
         ts: Date.now(),
+        agent: claudeAgent(parentToolUseId),
       },
     });
   }
@@ -354,8 +417,29 @@ class ClaudeEventParser {
           ...(parentToolUseId ? { parentToolUseId } : {}),
         },
         ts: Date.now(),
+        agent: claudeAgent(parentToolUseId),
       },
     });
+    if (isSubagentTool(block.name)) {
+      const agent = spawnedClaudeAgent(block.id, block.input, parentToolUseId);
+      const details =
+        typeof block.input === "object" && block.input !== null
+          ? (block.input as Record<string, unknown>)
+          : {};
+      content.events.push({
+        tag: "progress",
+        progress: {
+          id: `agent-spawn-${block.id}`,
+          kind: "agent-spawn",
+          payload: {
+            agent,
+            ...(typeof details.prompt === "string" ? { prompt: details.prompt } : {}),
+          },
+          agent: claudeAgent(parentToolUseId),
+          ts: Date.now(),
+        },
+      });
+    }
   }
 
   private recordTurn(
@@ -376,6 +460,7 @@ class ClaudeEventParser {
       tools: [],
       asked: false,
       thought: false,
+      agent: content.agent,
     };
     this.pendingTurn.usage = usage;
     this.pendingTurn.asked ||= content.asked;
@@ -451,6 +536,7 @@ class ClaudeEventParser {
         kind: "usage",
         payload: { ...this.pendingTurn.usage, ...(label ? { label } : {}) },
         ts: Date.now(),
+        agent: this.pendingTurn.agent,
       },
     };
     this.pendingTurn = null;

package/src/drivers/codex/app-events.ts

@@ -1,5 +1,7 @@
 import {
+  type AgentAttribution,
   type ParsedDriverEvent,
+  type Progress,
   type TokenUsage,
   TURN_GLYPHS,
   toolResultProgress,
@@ -20,8 +22,98 @@ const resultText = (content: unknown): string => {
     .join("\n");
 };
 
+const agentCompletionStatus = (status: unknown): "completed" | "failed" | "cancelled" => {
+  if (status === "failed") return "failed";
+  if (status === "cancelled" || status === "interrupted") return "cancelled";
+  return "completed";
+};
+
+export const appAgentLifecycleProgress = (
+  item: RpcRecord,
+  phase: "started" | "completed",
+  actor?: AgentAttribution,
+  ts = Date.now(),
+): Progress[] => {
+  if (item.type !== "collabAgentToolCall" || typeof item.id !== "string") return [];
+  const receiverThreadIds = Array.isArray(item.receiverThreadIds)
+    ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
+    : [];
+  const completionStatus = agentCompletionStatus(item.status);
+  return receiverThreadIds.map((receiverThreadId) =>
+    phase === "started"
+      ? {
+          id: `agent-spawn-${item.id}-${receiverThreadId}`,
+          kind: "agent-spawn",
+          payload: {
+            agent: {
+              id: receiverThreadId,
+              role: "subagent",
+              parentId: actor?.id ?? "codex",
+              spawnToolUseId: item.id as string,
+              ...(typeof item.model === "string" ? { model: item.model } : {}),
+            },
+            ...(typeof item.prompt === "string" ? { prompt: item.prompt } : {}),
+          },
+          ...(actor ? { agent: actor } : {}),
+          ts,
+        }
+      : {
+          id: `agent-completed-${item.id}-${receiverThreadId}`,
+          kind: "agent-completed",
+          payload: {
+            agentId: receiverThreadId,
+            status: completionStatus,
+          },
+          ...(actor ? { agent: actor } : {}),
+          ts,
+        },
+  );
+};
+
 export type RpcId = string | number;
 export type RpcRecord = Record<string, unknown>;
+
+export class CodexSubagentTracker {
+  private readonly spawnToolByThread = new Map<string, string>();
+  private readonly parentAgentByThread = new Map<string, string>();
+
+  record(
+    item: RpcRecord | null,
+    spawningThreadId: string | undefined,
+    mainThreadId: string | null,
+  ): void {
+    if (item?.type !== "collabAgentToolCall" || typeof item.id !== "string") return;
+    const receiverThreadIds = Array.isArray(item.receiverThreadIds)
+      ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
+      : [];
+    for (const receiverThreadId of receiverThreadIds) {
+      this.spawnToolByThread.set(receiverThreadId, item.id);
+      if (spawningThreadId) {
+        this.parentAgentByThread.set(
+          receiverThreadId,
+          spawningThreadId === mainThreadId ? "codex" : spawningThreadId,
+        );
+      }
+    }
+  }
+
+  attribution(
+    threadId: string | undefined,
+    mainThreadId: string | null,
+  ): AgentAttribution | undefined {
+    if (!threadId) return undefined;
+    const spawnToolUseId = this.spawnToolByThread.get(threadId);
+    if (!spawnToolUseId) {
+      return { id: threadId === mainThreadId ? "codex" : threadId, role: "main" };
+    }
+    return {
+      id: threadId,
+      role: "subagent",
+      parentId: this.parentAgentByThread.get(threadId) ?? mainThreadId ?? "codex",
+      spawnToolUseId,
+    };
+  }
+}
 export type AppTurnFacets = {
   tools: string[];
   asked: boolean;
@@ -131,8 +223,10 @@ const appItemResultText = (item: RpcRecord, error: RpcRecord | null, content: un
 export const appItemProgress = (
   item: RpcRecord,
   phase: "started" | "completed",
-  parentToolUseId?: string,
+  attribution?: AgentAttribution | string,
 ): ParsedDriverEvent | null => {
+  const agent = typeof attribution === "string" ? undefined : attribution;
+  const parentToolUseId = typeof attribution === "string" ? attribution : agent?.spawnToolUseId;
   const id = typeof item.id === "string" ? item.id : `${String(item.type)}-${Date.now()}`;
   if (item.type === "agentMessage" && phase === "completed" && typeof item.text === "string") {
     return {
@@ -142,6 +236,7 @@ export const appItemProgress = (
         kind: "agent-message",
         payload: { text: item.text, ...(parentToolUseId ? { parentToolUseId } : {}) },
         ts: Date.now(),
+        ...(agent ? { agent } : {}),
       },
     };
   }
@@ -162,6 +257,7 @@ export const appItemProgress = (
           ...(parentToolUseId ? { parentToolUseId } : {}),
         },
         ts: Date.now(),
+        ...(agent ? { agent } : {}),
       },
     };
   }
@@ -184,6 +280,7 @@ export const appItemProgress = (
           ...(parentToolUseId ? { parentToolUseId } : {}),
         },
         ts: Date.now(),
+        ...(agent ? { agent } : {}),
       },
     };
   }
@@ -199,20 +296,23 @@ export const appItemProgress = (
     error !== null;
   return {
     tag: "progress",
-    progress: toolResultProgress(
-      0,
-      {
-        name: tool,
-        toolUseId: id,
-        content,
-        ...(asRecord(structuredContent)
-          ? { structuredContent: asRecord(structuredContent) ?? {} }
-          : {}),
-        isError,
-        text: appItemResultText(item, error, content),
-        ...(parentToolUseId ? { parentToolUseId } : {}),
-      },
-      Date.now(),
-    ),
+    progress: {
+      ...toolResultProgress(
+        0,
+        {
+          name: tool,
+          toolUseId: id,
+          content,
+          ...(asRecord(structuredContent)
+            ? { structuredContent: asRecord(structuredContent) ?? {} }
+            : {}),
+          isError,
+          text: appItemResultText(item, error, content),
+          ...(parentToolUseId ? { parentToolUseId } : {}),
+        },
+        Date.now(),
+      ),
+      ...(agent ? { agent } : {}),
+    },
   };
 };

package/src/drivers/codex/index.ts

@@ -24,11 +24,13 @@ import { copyRegularFileNoFollow } from "../shared/secure-copy.js";
 import { subprocessDriver } from "../subprocess/index.js";
 import {
   type AppTurnFacets,
+  appAgentLifecycleProgress,
   appItemProgress,
   appItemToolName,
   appTurnLabel,
   appUsage,
   asRecord,
+  CodexSubagentTracker,
   messageOf,
   type RpcId,
   type RpcRecord,
@@ -316,7 +318,7 @@ class CodexAppServerDriverImpl implements AgentDriver {
     { resolve: (result: RpcRecord) => void; reject: (error: Error) => void }
   >();
   private readonly interrupts = new Map<string, { id: RpcId; method: string; params: RpcRecord }>();
-  private readonly subagentParents = new Map<string, string>();
+  private readonly subagents = new CodexSubagentTracker();
   private readonly turnFacets = new Map<string, AppTurnFacets>();
   private interruptCount = 0;
   private idleTimer: NodeJS.Timeout | null = null;
@@ -702,22 +704,22 @@ class CodexAppServerDriverImpl implements AgentDriver {
     const toolName = item ? appItemToolName(item) : null;
     if (toolName && !facets.tools.includes(toolName)) facets.tools.push(toolName);
     if (turnId) this.turnFacets.set(turnId, facets);
-    this.recordSubagentParents(item);
-    const parentToolUseId =
-      typeof params.threadId === "string" ? this.subagentParents.get(params.threadId) : undefined;
+    const threadId = typeof params.threadId === "string" ? params.threadId : undefined;
+    this.subagents.record(item, threadId, this.threadId);
+    const agent = this.subagents.attribution(threadId, this.threadId);
     const event = item
-      ? appItemProgress(item, method === "item/started" ? "started" : "completed", parentToolUseId)
+      ? appItemProgress(item, method === "item/started" ? "started" : "completed", agent)
       : null;
     if (event?.tag === "progress") await this.emitter.emit("progress", event.progress);
-  }
-
-  private recordSubagentParents(item: RpcRecord | null): void {
-    if (item?.type !== "collabAgentToolCall" || typeof item.id !== "string") return;
-    const receiverThreadIds = Array.isArray(item.receiverThreadIds)
-      ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
-      : [];
-    for (const receiverThreadId of receiverThreadIds) {
-      this.subagentParents.set(receiverThreadId, item.id);
+    if (item) {
+      for (const lifecycle of appAgentLifecycleProgress(
+        item,
+        method === "item/started" ? "started" : "completed",
+        agent,
+        event?.tag === "progress" ? event.progress.ts : Date.now(),
+      )) {
+        await this.emitter.emit("progress", lifecycle);
+      }
     }
   }
 

package/src/drivers/subprocess/index.ts

@@ -82,6 +82,7 @@ const KILL_GRACE_MS = 5_000;
 const DEFAULT_MAX_LINE_BYTES = 10_000_000;
 const MAX_STDERR_TAIL_CHARS = 8_192;
 const DEFAULT_MAX_ARTIFACT_FILES = 500;
+const MAX_FIXTURE_VALUES_BYTES = 64 * 1024;
 const ARTIFACT_SNAPSHOT_MODULE = fileURLToPath(
   new URL("../../core/artifact-snapshot.js", import.meta.url),
 );
@@ -139,6 +140,10 @@ class SubprocessDriverImpl implements AgentDriver {
 
   async start(ctx: StartContext): Promise<ReadyState> {
     this.artifactsDir = ctx.artifactsDir;
+    const fixtureValues = JSON.stringify(ctx.fixtureValues ?? []);
+    if (Buffer.byteLength(fixtureValues) > MAX_FIXTURE_VALUES_BYTES) {
+      throw new Error(`subprocess fixture values exceed ${MAX_FIXTURE_VALUES_BYTES} bytes`);
+    }
     const managed = spawnManagedProcess(this.config.command, this.config.args ?? [], {
       cwd: this.config.cwd ?? ctx.runDir,
       env: {
@@ -147,6 +152,7 @@ class SubprocessDriverImpl implements AgentDriver {
         ...(ctx.caseId ? { LUX_CASE_ID: ctx.caseId } : {}),
         LUX_RUN_DIR: ctx.runDir,
         LUX_ARTIFACTS_DIR: ctx.artifactsDir,
+        LUX_FIXTURE_VALUES: fixtureValues,
         LUX_PROMPT: ctx.prompt,
         LUX_ARTIFACT_SNAPSHOT_MODULE: ARTIFACT_SNAPSHOT_MODULE,
       },

package/src/index.ts

@@ -16,16 +16,15 @@ export {
   PersonaAnswererConfigSchema,
   personaAnswerer,
 } from "./answerers/persona/index.js";
-export {
-  type ScriptedAnswererConfig,
-  ScriptedAnswererConfigSchema,
-  scriptedAnswerer,
-} from "./answerers/scripted/index.js";
 export {
   type TerminalAnswererConfig,
   TerminalAnswererConfigSchema,
   terminalAnswerer,
 } from "./answerers/terminal/index.js";
+export {
+  type BuiltinAssertion,
+  defineEvalCase,
+} from "./assertions/authoring.js";
 // Bundled deterministic and judge assertions.
 export {
   type AgentScope,
@@ -88,6 +87,8 @@ export {
   registerCoreAssertions,
   runSucceededAssertion,
   subagentToolCalls,
+  type TimeToFirstActionUnderConfig,
+  TimeToFirstActionUnderConfigSchema,
   type TokensUnderConfig,
   TokensUnderConfigSchema,
   type ToolCall,
@@ -105,6 +106,7 @@ export {
   ToolResultIsErrorConfigSchema,
   type ToolResultJsonPointerEqualsConfig,
   ToolResultJsonPointerEqualsConfigSchema,
+  timeToFirstActionUnderAssertion,
   tokensUnderAssertion,
   toolCallCountAssertion,
   toolCalledAssertion,
@@ -142,6 +144,28 @@ export {
   rubricAssertion,
 } from "./assertions/rubric/index.js";
 // Authoring and extension contracts.
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
+export {
+  type AgentActivity,
+  AgentActivitySchema,
+  summarizeAgentActivity,
+} from "./core/agent-activity.js";
+export {
+  AGENT_COMPLETED_KIND,
+  AGENT_SPAWN_KIND,
+  type AgentCompletedPayload,
+  AgentCompletedPayloadSchema,
+  type AgentSpawnPayload,
+  AgentSpawnPayloadSchema,
+} from "./core/agent-events.js";
 export {
   defineGraderAnnotations,
   type GraderAnnotation,
@@ -185,6 +209,7 @@ export {
   type AssertionResult,
   AssertionResultSchema,
   defineAssertion,
+  defineAssertionRegistry,
   type EvaluateContext,
   type GradingAssertionEntry,
   GradingAssertionEntrySchema,
@@ -198,8 +223,9 @@ export {
   AssertionSchema,
   type DriverRef,
   DriverRefSchema,
-  defineEvalCase,
   type EvalCase,
+  type EvalCaseAuthoringOptions,
+  type EvalCaseDefinition,
   EvalCaseSchema,
   type FixturesRef,
   FixturesRefSchema,
@@ -226,8 +252,23 @@ export {
   ResolvedFixturesSchema,
   type StartContext,
 } from "./core/driver.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
 export {
+  type AgentAttribution,
+  AgentAttributionSchema,
   type Answer,
   AnswerSchema,
   ApproveAnswerSchema,
@@ -300,6 +341,9 @@ export {
 export {
   type CaseLifecycleHookContext,
   defineLifecycleHooks,
+  type FixtureValue,
+  type FixtureValueBinding,
+  FixtureValueSchema,
   type LifecycleHookCleanup,
   type LifecycleHookSetupResult,
   type LifecycleHooksPlugin,

package/src/orchestrator/annotation-store.ts

@@ -57,6 +57,7 @@ const recoverStaleLock = async (lockPath: string): Promise<boolean> => {
 const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>> => {
   const lockPath = `${path}.lock`;
   const deadline = Date.now() + LOCK_WAIT_MS;
+  let attemptedRecovery = false;
   while (true) {
     try {
       const owner = await publishOwnedLock(lockPath, {
@@ -69,7 +70,10 @@ const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>>
       };
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-      if (await recoverStaleLock(lockPath)) continue;
+      if (!attemptedRecovery) {
+        attemptedRecovery = true;
+        if (await recoverStaleLock(lockPath)) continue;
+      }
       if (Date.now() >= deadline) {
         throw new Error(`timed out waiting to update grader annotations: ${path}`);
       }

package/src/orchestrator/case-extensions.ts

@@ -0,0 +1,44 @@
+import { evalCaseExtensions } from "../core/case-authoring-state.js";
+import type {
+  EvalCase,
+  FixtureValueBinding,
+  LifecycleHooksPlugin,
+  Registry,
+} from "../core/index.js";
+
+export const registryForEvalCase = (registry: Registry, evalCase: EvalCase): Registry => {
+  let resolved = registry;
+  for (const plugin of evalCaseExtensions(evalCase).assertions) {
+    const existing = resolved.getAssertion(plugin.type);
+    if (existing === plugin) continue;
+    if (existing) {
+      throw new TypeError(
+        `case-local assertion '${plugin.type}' conflicts with a registered implementation`,
+      );
+    }
+    resolved = resolved.withAssertion(plugin);
+  }
+  return resolved;
+};
+
+export const fixturesForEvalCase = (evalCase: EvalCase): readonly LifecycleHooksPlugin[] =>
+  evalCaseExtensions(evalCase).fixtures;
+
+export const valuesForFixtures = (
+  fixtures: readonly LifecycleHooksPlugin[],
+): FixtureValueBinding[] =>
+  fixtures.flatMap(({ name, value }) => (value === undefined ? [] : [{ name, value }]));
+
+export const mergeLifecycleFixtures = (
+  ...groups: readonly (readonly LifecycleHooksPlugin[])[]
+): LifecycleHooksPlugin[] => {
+  const fixtures = new Map<string, LifecycleHooksPlugin>();
+  for (const fixture of groups.flat()) {
+    const existing = fixtures.get(fixture.name);
+    if (existing && existing !== fixture) {
+      throw new TypeError(`conflicting lifecycle fixture definitions for '${fixture.name}'`);
+    }
+    fixtures.set(fixture.name, fixture);
+  }
+  return [...fixtures.values()];
+};

package/src/orchestrator/compare.ts

@@ -2,6 +2,9 @@ import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  type ActionTiming,
+  type AgentActivity,
+  type DriverMetrics,
   emptyRunUsage,
   type GradingReport,
   GradingReportSchema,
@@ -48,6 +51,9 @@ export type RunComparison = {
     exitReason: { a: string; b: string };
     durationMsA: number | null;
     durationMsB: number | null;
+    actionTiming: { a: ActionTiming; b: ActionTiming };
+    driverMetrics: { a: DriverMetrics; b: DriverMetrics };
+    agentActivity: { a: AgentActivity[]; b: AgentActivity[] };
     interruptCounts: {
       a: { ask: number; approve: number; custom: number };
       b: { ask: number; approve: number; custom: number };
@@ -260,6 +266,18 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
           },
           durationMsA: durationFromMeta(a.runJson),
           durationMsB: durationFromMeta(b.runJson),
+          actionTiming: {
+            a: a.runJson?.actionTiming ?? {},
+            b: b.runJson?.actionTiming ?? {},
+          },
+          driverMetrics: {
+            a: a.runJson?.driverMetrics ?? {},
+            b: b.runJson?.driverMetrics ?? {},
+          },
+          agentActivity: {
+            a: a.runJson?.agentActivity ?? [],
+            b: b.runJson?.agentActivity ?? [],
+          },
           interruptCounts: {
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },

package/src/orchestrator/config.ts

@@ -4,7 +4,6 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
@@ -182,7 +181,6 @@ const builtinDrivers: DriverPlugin<never>[] = [
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {