@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.5

package/src/cli/view.ts

@@ -0,0 +1,168 @@
+import { resolve } from "node:path";
+import { InvalidArgumentError } from "commander";
+import {
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  formatRunDetail,
+  formatRunsTable,
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  isExperimentRun,
+  listExperimentRuns,
+  listRuns,
+  listSuiteRuns,
+  loadExperimentRun,
+  loadLuxConfig,
+  loadRunDetail,
+  loadSuiteRun,
+} from "../orchestrator/index.js";
+import { openPreviewUrl, startPreviewServer } from "../previewer/server.js";
+import { abortOnInterrupt, ensureDirectory, failWith } from "./command-runtime.js";
+
+const SIGINT_EXIT_CODE = 130;
+
+export type ViewOptions = {
+  runsRoot?: string;
+  case?: string;
+  suites?: boolean;
+  experiments?: boolean;
+  web?: boolean;
+  port?: number;
+  open?: boolean;
+  json?: boolean;
+};
+
+const previewKind = async (
+  initialPath: string | undefined,
+  options: ViewOptions,
+): Promise<"run" | "suite" | "experiment" | undefined> => {
+  if (initialPath) {
+    if (isExperimentRun(initialPath)) return "experiment";
+    if (await loadSuiteRun(initialPath)) return "suite";
+    return "run";
+  }
+  if (options.experiments) return "experiment";
+  if (options.suites) return "suite";
+  return undefined;
+};
+
+const webViewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.json) {
+    throw new InvalidArgumentError("--web and --json are mutually exclusive");
+  }
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  const initialPath = runDirArg ? resolve(runDirArg) : undefined;
+  if (initialPath && !(await ensureDirectory("view", initialPath))) return;
+  const initialKind = await previewKind(initialPath, options);
+  const controller = abortOnInterrupt();
+  const preview = await startPreviewServer({
+    runsRoot,
+    ...(options.port !== undefined ? { port: options.port } : {}),
+    ...(initialPath ? { initialPath } : {}),
+    ...(initialKind ? { initialKind } : {}),
+    ...(options.case ? { caseFilter: options.case } : {}),
+    signal: controller.signal,
+  });
+  process.stdout.write(`Lux Review: ${preview.url}\n`);
+  process.stdout.write(`Reading evidence from ${runsRoot}\n`);
+  if (options.open !== false) {
+    try {
+      await openPreviewUrl(preview.url);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      process.stderr.write(`lux view: could not open a browser (${message})\n`);
+    }
+  }
+  await preview.closed;
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+};
+
+export const viewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.web) {
+    await webViewCommand(runDirArg, options);
+    return;
+  }
+  if (options.port !== undefined || options.open === false) {
+    throw new InvalidArgumentError("--port and --no-open require --web");
+  }
+  if (runDirArg) {
+    const path = resolve(runDirArg);
+    if (!(await ensureDirectory("view", path))) return;
+    if (isExperimentRun(path)) {
+      const experiment = await loadExperimentRun(path);
+      if (!experiment) {
+        process.stderr.write(`lux view: invalid experiment at ${path}\n`);
+        failWith(1);
+        return;
+      }
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
+          : `${formatExperimentRun(path, experiment)}\n`,
+      );
+      return;
+    }
+    const suite = await loadSuiteRun(path);
+    if (suite) {
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
+          : `${formatSuiteRunDetail(path, suite)}\n`,
+      );
+      return;
+    }
+    const detail = await loadRunDetail(path);
+    process.stdout.write(
+      options.json ? `${JSON.stringify(detail, null, 2)}\n` : `${formatRunDetail(detail)}\n`,
+    );
+    return;
+  }
+
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  if (options.experiments) {
+    const experiments = await listExperimentRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(experiments, null, 2)}\n`
+        : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
+    );
+    return;
+  }
+  if (options.suites) {
+    const suites = await listSuiteRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(suites, null, 2)}\n`
+        : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
+    );
+    return;
+  }
+  const all = await listRuns(runsRoot);
+  const runs = options.case ? all.filter((run) => run.caseSlug === options.case) : all;
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+    return;
+  }
+  if (!options.case) {
+    process.stdout.write(`suite runs at ${runsRoot}:\n`);
+    process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
+  }
+  process.stdout.write(`runs at ${runsRoot}:\n`);
+  process.stdout.write(`${formatRunsTable(runs)}\n`);
+};
+
+export const parsePreviewPort = (value: string): number => {
+  const port = Number(value);
+  if (!Number.isInteger(port) || port < 0 || port > 65_535) {
+    throw new InvalidArgumentError("port must be an integer from 0 to 65535");
+  }
+  return port;
+};

package/src/core/action-events.ts

@@ -0,0 +1,59 @@
+import { z } from "zod";
+import type { RunEvent } from "./events.js";
+
+export const ActionCategorySchema = z.enum([
+  "query",
+  "mutation",
+  "durable-mutation",
+  "execution",
+  "interaction",
+]);
+export type ActionCategory = z.infer<typeof ActionCategorySchema>;
+
+export const ActionProgressPayloadSchema = z
+  .object({
+    name: z.string(),
+    category: ActionCategorySchema,
+  })
+  .strict();
+export type ActionProgressPayload = z.infer<typeof ActionProgressPayloadSchema>;
+
+export const ActionTimingSchema = z.object({
+  timeToFirstActionMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstMutationMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstDurableMutationMs: z.exactOptional(z.number().nonnegative()),
+});
+export type ActionTiming = z.infer<typeof ActionTimingSchema>;
+
+export const actionTiming = (events: readonly RunEvent[]): ActionTiming => {
+  const startEvent = events.find(
+    (event) => event.type === "lifecycle" && event.stage === "start-driver",
+  );
+  const startedAt = startEvent?.type === "lifecycle" ? startEvent.ts : undefined;
+  if (startedAt === undefined) return {};
+
+  const actions = events.flatMap((event) => {
+    if (event.type !== "progress" || event.kind !== "action") return [];
+    const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+    return payload ? [{ ...payload, ts: event.ts }] : [];
+  });
+  const elapsed = (category: "any" | "mutation" | "durable-mutation") => {
+    const timestamps = actions.flatMap(({ category: candidate, ts }) => {
+      if (category === "any") return [ts];
+      if (category === "mutation") {
+        return candidate === "mutation" || candidate === "durable-mutation" ? [ts] : [];
+      }
+      return candidate === category ? [ts] : [];
+    });
+    return timestamps.length > 0 ? Math.max(0, Math.min(...timestamps) - startedAt) : undefined;
+  };
+
+  const timeToFirstActionMs = elapsed("any");
+  const timeToFirstMutationMs = elapsed("mutation");
+  const timeToFirstDurableMutationMs = elapsed("durable-mutation");
+  return {
+    ...(timeToFirstActionMs === undefined ? {} : { timeToFirstActionMs }),
+    ...(timeToFirstMutationMs === undefined ? {} : { timeToFirstMutationMs }),
+    ...(timeToFirstDurableMutationMs === undefined ? {} : { timeToFirstDurableMutationMs }),
+  };
+};

package/src/core/artifact-evidence.ts

@@ -7,12 +7,15 @@ import type { Run } from "./run.js";
 const DEFAULT_ARTIFACT_READ_BYTES = 5 * 1024 * 1024;
 const MAX_ARTIFACT_READ_BYTES = 10 * 1024 * 1024;
 
+export type RunArtifactIssueReason = "absent" | "unreadable";
+
 export type RunArtifactInspection = {
   target: string;
   root: string;
   artifact?: Artifact;
   stats?: Stats;
   issue?: string;
+  reason?: RunArtifactIssueReason;
 };
 
 type RunArtifactEvidence = {
@@ -82,6 +85,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: runArtifactsRoot(run),
       issue: `${authoredPath} contains a null byte`,
+      reason: "unreadable",
     };
   }
   const matching = run.artifacts.filter((candidate) => candidate.path === authoredPath);
@@ -91,6 +95,7 @@ export const inspectRunArtifact = async (
       target: resolve(root, authoredPath),
       root,
       issue: `${authoredPath} was not captured in the frozen artifact index`,
+      reason: "absent",
     };
   }
   if (matching.some((candidate) => candidate.root !== undefined && !isAbsolute(candidate.root))) {
@@ -98,6 +103,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} declares a non-absolute artifact root`,
+      reason: "unreadable",
     };
   }
   const artifact = matching[0];
@@ -108,6 +114,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} is ambiguous across multiple artifact roots`,
+      reason: "unreadable",
     };
   }
   // A non-default root is trusted only for an exact artifact-index entry.
@@ -116,7 +123,12 @@ export const inspectRunArtifact = async (
   const root = matchingRoots[0] ?? defaultArtifactsRoot(run);
   const target = resolve(root, authoredPath);
   if (pathEscapesRoot(root, target)) {
-    return { target, root, issue: `${authoredPath} escapes the captured artifact root` };
+    return {
+      target,
+      root,
+      issue: `${authoredPath} escapes the captured artifact root`,
+      reason: "unreadable",
+    };
   }
 
   const path = relative(root, target);
@@ -134,7 +146,13 @@ export const inspectRunArtifact = async (
     } catch (error) {
       const code = (error as NodeJS.ErrnoException).code;
       if (code === "ENOENT" || code === "ENOTDIR") {
-        return { target, root, ...identified, issue: `${authoredPath} not found` };
+        return {
+          target,
+          root,
+          ...identified,
+          issue: `${authoredPath} not found`,
+          reason: "absent",
+        };
       }
       throw error;
     }
@@ -144,16 +162,17 @@ export const inspectRunArtifact = async (
         root,
         ...identified,
         issue: `${authoredPath} resolves through a symlink`,
+        reason: "unreadable",
       };
     }
     if (index < components.length - 1 && !stats.isDirectory()) {
-      return { target, root, ...identified, issue: `${authoredPath} not found` };
+      return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
     }
     if (index === components.length - 1) {
       return { target, root, ...identified, stats };
     }
   }
-  return { target, root, ...identified, issue: `${authoredPath} not found` };
+  return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
 };
 
 export type ReadRunArtifactTextOptions = {
@@ -247,3 +266,27 @@ export const readRunArtifactText = async (
 ): Promise<string> => {
   return (await readRunArtifactBytes(run, authoredPath, options)).toString("utf8");
 };
+
+/**
+ * Read optional artifact evidence. Returns null when the artifact was never
+ * captured, and still throws when one exists but is unsafe to consume, so a
+ * traversal or symlink violation is never mistaken for missing evidence.
+ */
+export const readRunArtifactTextIfPresent = async (
+  run: RunArtifactEvidence,
+  authoredPath: string,
+  options: ReadRunArtifactTextOptions = {},
+): Promise<string | null> => {
+  const inspected = await inspectRunArtifact(run, authoredPath);
+  if (inspected.reason === "absent") return null;
+  if (!inspected.stats?.isFile()) {
+    throw new Error(
+      inspected.issue ??
+        (inspected.stats ? `${authoredPath} is not a file` : `${authoredPath} not found`),
+    );
+  }
+  const maxBytes = options.maxBytes ?? DEFAULT_ARTIFACT_READ_BYTES;
+  return (
+    await readInspectedFileBytes(inspected.target, inspected.stats, maxBytes, authoredPath)
+  ).toString("utf8");
+};

package/src/core/assertion.ts

@@ -100,6 +100,13 @@ export const assertionPluginIssue = (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
 export const defineAssertion = <S extends z.ZodObject>(
   definition: AssertionPluginDefinition<S>,
 ): AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>> => {
@@ -141,11 +148,19 @@ export const GradingAssertionEntrySchema = z.object({
 });
 export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
 
+const SkippedGradingAssertionSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+  reason: z.literal("judge-disabled"),
+});
+
 export const GradingReportSchema = z.object({
   schemaVersion: z.literal(1),
   casePassed: z.boolean().nullable(),
   caseScore: z.number().nullable(),
   assertions: z.array(GradingAssertionEntrySchema),
+  skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
   /**
    * Present when the subject run or grading did not complete. Assertions may
    * still contain partial diagnostics, but the report is not a clean verdict.

package/src/core/driver-metrics.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const DriverMetricUnitSchema = z.enum([
+  "number",
+  "count",
+  "milliseconds",
+  "bytes",
+  "tokens",
+  "usd",
+  "ratio",
+  "percent",
+  "score",
+]);
+export type DriverMetricUnit = z.infer<typeof DriverMetricUnitSchema>;
+
+export const DriverMetricDescriptorSchema = z
+  .object({
+    unit: DriverMetricUnitSchema.default("number"),
+    label: z.string().min(1).max(120).optional(),
+    better: z.enum(["lower", "higher", "neutral"]).default("neutral"),
+  })
+  .strict();
+export type DriverMetricDescriptor = z.infer<typeof DriverMetricDescriptorSchema>;
+
+export const DriverMetricSchema = DriverMetricDescriptorSchema.extend({
+  value: z.number().finite(),
+}).strict();
+export type DriverMetric = z.infer<typeof DriverMetricSchema>;
+
+export const DriverMetricsSchema = z.record(z.string().min(1).max(120), DriverMetricSchema);
+export type DriverMetrics = z.infer<typeof DriverMetricsSchema>;
+
+const ReportedDriverMetricSchema = z.union([z.number().finite(), DriverMetricSchema]);
+export const ReportedDriverMetricsSchema = z.record(
+  z.string().min(1).max(120),
+  ReportedDriverMetricSchema,
+);
+export type ReportedDriverMetrics = z.input<typeof ReportedDriverMetricsSchema>;
+
+export const normalizeDriverMetrics = (value: unknown): DriverMetrics => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value).flatMap(([name, metric]) => {
+      if (name.length === 0 || name.length > 120) return [];
+      const parsed = DriverMetricSchema.safeParse(
+        typeof metric === "number" ? { value: metric } : metric,
+      ).data;
+      return parsed ? [[name, parsed]] : [];
+    }),
+  );
+};

package/src/core/events.ts

@@ -69,8 +69,8 @@ export const DoneSchema = z.object({
   exitReason: z.literal("done"),
   /**
    * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
-   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
-   * for domain-specific measurements.
+   * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
+   * preserving room for other domain-specific evidence.
    */
   summary: z.unknown().optional(),
 });
@@ -82,6 +82,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
 export const ErrorEventSchema = z.object({
   exitReason: ErrorEventReasonSchema,
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
 

package/src/core/experiment.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
 import { ConfidenceConfigSchema } from "./statistics.js";
 import { TokenUsageSchema } from "./usage.js";
 import { formatZodIssues } from "./zod.js";
@@ -415,6 +416,7 @@ export const CandidateEvaluationSchema = z.object({
   passRate: z.number().min(0).max(1),
   casePassRates: z.record(z.string(), z.number().min(0).max(1)),
   metrics: z.record(z.string(), z.number()),
+  metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
   metricCalls: z.exactOptional(z.number().int().nonnegative()),
   judgeCalls: z.exactOptional(z.number().int().nonnegative()),
   unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),

package/src/core/index.ts

@@ -1,3 +1,4 @@
+export * from "./action-events.js";
 export * from "./annotation.js";
 export * from "./answerer.js";
 export * from "./artifact-evidence.js";
@@ -6,6 +7,7 @@ export * from "./assertion.js";
 export * from "./case.js";
 export * from "./conversation-turn.js";
 export * from "./driver.js";
+export * from "./driver-metrics.js";
 export * from "./emitter.js";
 export * from "./events.js";
 export * from "./exec.js";

package/src/core/run.ts

@@ -1,6 +1,8 @@
 import { z } from "zod";
+import { ActionTimingSchema } from "./action-events.js";
 import { EvalCaseSchema } from "./case.js";
 import { ArtifactSchema } from "./driver.js";
+import { DriverMetricsSchema } from "./driver-metrics.js";
 import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
 import { ToolOutcomeSummarySchema } from "./tool-events.js";
 import { RunUsageSchema } from "./usage.js";
@@ -112,6 +114,8 @@ export const RunMetadataSchema = z.object({
     }),
   ),
   cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
+  /** Driver-defined scalar measurements with display and optimization semantics. */
+  driverMetrics: z.exactOptional(DriverMetricsSchema),
   interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
   /** Legacy raw count of progress events whose kind is `tool-call`. */
   toolCallCount: z.exactOptional(z.number()),
@@ -124,6 +128,8 @@ export const RunMetadataSchema = z.object({
   ),
   /** Wall-clock per phase, derived from the lifecycle event stream. */
   phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
+  /** Driver-reported semantic action latency, measured from `start-driver`. */
+  actionTiming: z.exactOptional(ActionTimingSchema),
   /**
    * Harness/infrastructure failure that invalidates this run as candidate
    * evidence. A driver-reported subject failure intentionally leaves this

package/src/core/wire.ts

@@ -47,6 +47,7 @@ const ErrorEventSchema = z.object({
   type: z.literal("error"),
   exitReason: ErrorEventReasonSchema.default("error"),
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 
 export const WireEventSchema = z.discriminatedUnion("type", [
@@ -85,7 +86,10 @@ const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
   if (event.type === "done") {
     return { tag: "done", done: { exitReason: "done", summary: event.summary } };
   }
-  return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
+  return {
+    tag: "error",
+    error: { exitReason: event.exitReason, reason: event.reason, summary: event.summary },
+  };
 };
 
 /**

package/src/index.ts

@@ -16,11 +16,6 @@ export {
   PersonaAnswererConfigSchema,
   personaAnswerer,
 } from "./answerers/persona/index.js";
-export {
-  type ScriptedAnswererConfig,
-  ScriptedAnswererConfigSchema,
-  scriptedAnswerer,
-} from "./answerers/scripted/index.js";
 export {
   type TerminalAnswererConfig,
   TerminalAnswererConfigSchema,
@@ -92,6 +87,8 @@ export {
   registerCoreAssertions,
   runSucceededAssertion,
   subagentToolCalls,
+  type TimeToFirstActionUnderConfig,
+  TimeToFirstActionUnderConfigSchema,
   type TokensUnderConfig,
   TokensUnderConfigSchema,
   type ToolCall,
@@ -109,6 +106,7 @@ export {
   ToolResultIsErrorConfigSchema,
   type ToolResultJsonPointerEqualsConfig,
   ToolResultJsonPointerEqualsConfigSchema,
+  timeToFirstActionUnderAssertion,
   tokensUnderAssertion,
   toolCallCountAssertion,
   toolCalledAssertion,
@@ -146,6 +144,15 @@ export {
   rubricAssertion,
 } from "./assertions/rubric/index.js";
 // Authoring and extension contracts.
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
 export {
   defineGraderAnnotations,
   type GraderAnnotation,
@@ -232,6 +239,19 @@ export {
   ResolvedFixturesSchema,
   type StartContext,
 } from "./core/driver.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
 export {
   type Answer,

package/src/orchestrator/compare.ts

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  type ActionTiming,
+  type DriverMetrics,
   emptyRunUsage,
   type GradingReport,
   GradingReportSchema,
@@ -48,6 +50,8 @@ export type RunComparison = {
     exitReason: { a: string; b: string };
     durationMsA: number | null;
     durationMsB: number | null;
+    actionTiming: { a: ActionTiming; b: ActionTiming };
+    driverMetrics: { a: DriverMetrics; b: DriverMetrics };
     interruptCounts: {
       a: { ask: number; approve: number; custom: number };
       b: { ask: number; approve: number; custom: number };
@@ -260,6 +264,14 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
           },
           durationMsA: durationFromMeta(a.runJson),
           durationMsB: durationFromMeta(b.runJson),
+          actionTiming: {
+            a: a.runJson?.actionTiming ?? {},
+            b: b.runJson?.actionTiming ?? {},
+          },
+          driverMetrics: {
+            a: a.runJson?.driverMetrics ?? {},
+            b: b.runJson?.driverMetrics ?? {},
+          },
           interruptCounts: {
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },

package/src/orchestrator/config.ts

@@ -4,7 +4,6 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
@@ -182,7 +181,6 @@ const builtinDrivers: DriverPlugin<never>[] = [
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {