@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.5

package/src/assertions/core/resource-checks.ts

@@ -44,6 +44,41 @@ export const latencyUnderAssertion: AssertionImpl<LatencyUnderConfig> = {
   },
 };
 
+export const TimeToFirstActionUnderConfigSchema = z
+  .object({
+    type: z.literal("time-to-first-action-under"),
+    maxMs: z.number().nonnegative(),
+    category: z.enum(["any", "mutation", "durable-mutation"]).default("any"),
+    onUnavailable: UnavailablePolicySchema.default("fail"),
+    ...baseFields,
+  })
+  .strict();
+export type TimeToFirstActionUnderConfig = z.infer<typeof TimeToFirstActionUnderConfigSchema>;
+
+export const timeToFirstActionUnderAssertion: AssertionImpl<TimeToFirstActionUnderConfig> = {
+  type: "time-to-first-action-under",
+  configSchema: TimeToFirstActionUnderConfigSchema,
+  evaluate: async (config, run) => {
+    const fields = {
+      any: "timeToFirstActionMs",
+      mutation: "timeToFirstMutationMs",
+      "durable-mutation": "timeToFirstDurableMutationMs",
+    } as const;
+    const field = fields[config.category];
+    const value = run.metadata.actionTiming?.[field];
+    if (value === undefined) {
+      return unavailable(`time to first ${config.category} action`, config.onUnavailable);
+    }
+    const passed = value <= config.maxMs;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${config.category} action at ${value}ms ${passed ? "<=" : ">"} ${config.maxMs}ms`,
+      detail: { category: config.category, timeToFirstActionMs: value },
+    };
+  },
+};
+
 export const TokensUnderConfigSchema = z
   .object({
     type: z.literal("tokens-under"),

package/src/cli/program.ts

@@ -24,25 +24,15 @@ import {
   formatDoctorReport,
   formatExperimentDecisionMarkdown,
   formatExperimentRun,
-  formatExperimentRunsTable,
-  formatRunDetail,
-  formatRunsTable,
   formatSuiteComparisonText,
-  formatSuiteRunDetail,
-  formatSuiteRunsTable,
   hasRegression,
   hasSuiteRegression,
-  isExperimentRun,
-  listExperimentRuns,
-  listRuns,
-  listSuiteRuns,
   loadEvalCase,
   loadExperimentCampaign,
   loadExperimentRun,
   loadFrozenRegradableRun,
   loadLuxConfig,
   loadRubricDefinition,
-  loadRunDetail,
   loadSuiteRun,
   loadVerifiedExperimentDecisionSnapshot,
   Orchestrator,
@@ -84,6 +74,7 @@ import {
   installSkills,
   type SkillsPlatform,
 } from "./skills.js";
+import { parsePreviewPort, viewCommand } from "./view.js";
 
 const requireFromHere = createRequire(import.meta.url);
 // From lib/cli/program.js (or src/cli/program.ts under test) to the package root.
@@ -496,88 +487,11 @@ const registerReportingCommands = (program: Command): void => {
     .option("--case <id>", "filter the run list to one case slug (list mode only)")
     .option("--suites", "list logical suite runs instead of individual runs")
     .option("--experiments", "list experiment runs instead of individual runs")
+    .option("--web", "open the local Lux Review web app")
+    .option("--port <n>", "listen on a specific local port in web mode", parsePreviewPort)
+    .option("--no-open", "start web mode without opening a browser")
     .option("--json", "emit machine-readable JSON")
-    .action(
-      async (
-        runDirArg: string | undefined,
-        options: {
-          runsRoot?: string;
-          case?: string;
-          suites?: boolean;
-          experiments?: boolean;
-          json?: boolean;
-        },
-      ) => {
-        if (runDirArg) {
-          const path = resolve(runDirArg);
-          if (!(await ensureDirectory("view", path))) return;
-          if (isExperimentRun(path)) {
-            const experiment = await loadExperimentRun(path);
-            if (!experiment) {
-              process.stderr.write(`lux view: invalid experiment at ${path}\n`);
-              failWith(1);
-              return;
-            }
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
-                : `${formatExperimentRun(path, experiment)}\n`,
-            );
-            return;
-          }
-          const suite = await loadSuiteRun(path);
-          if (suite) {
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
-                : `${formatSuiteRunDetail(path, suite)}\n`,
-            );
-            return;
-          }
-          const detail = await loadRunDetail(path);
-          if (options.json) {
-            process.stdout.write(`${JSON.stringify(detail, null, 2)}\n`);
-          } else {
-            process.stdout.write(`${formatRunDetail(detail)}\n`);
-          }
-          return;
-        }
-
-        const cwd = process.cwd();
-        const config = await loadLuxConfig(cwd);
-        const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
-        if (options.experiments) {
-          const experiments = await listExperimentRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(experiments, null, 2)}\n`
-              : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
-          );
-          return;
-        }
-        if (options.suites) {
-          const suites = await listSuiteRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(suites, null, 2)}\n`
-              : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
-          );
-          return;
-        }
-        const all = await listRuns(runsRoot);
-        const runs = options.case ? all.filter((r) => r.caseSlug === options.case) : all;
-        if (options.json) {
-          process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
-        } else {
-          if (!options.case) {
-            process.stdout.write(`suite runs at ${runsRoot}:\n`);
-            process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
-          }
-          process.stdout.write(`runs at ${runsRoot}:\n`);
-          process.stdout.write(`${formatRunsTable(runs)}\n`);
-        }
-      },
-    );
+    .action(viewCommand);
 
   program
     .command("compare <runA> <runB>")

package/src/cli/view.ts

@@ -0,0 +1,168 @@
+import { resolve } from "node:path";
+import { InvalidArgumentError } from "commander";
+import {
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  formatRunDetail,
+  formatRunsTable,
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  isExperimentRun,
+  listExperimentRuns,
+  listRuns,
+  listSuiteRuns,
+  loadExperimentRun,
+  loadLuxConfig,
+  loadRunDetail,
+  loadSuiteRun,
+} from "../orchestrator/index.js";
+import { openPreviewUrl, startPreviewServer } from "../previewer/server.js";
+import { abortOnInterrupt, ensureDirectory, failWith } from "./command-runtime.js";
+
+const SIGINT_EXIT_CODE = 130;
+
+export type ViewOptions = {
+  runsRoot?: string;
+  case?: string;
+  suites?: boolean;
+  experiments?: boolean;
+  web?: boolean;
+  port?: number;
+  open?: boolean;
+  json?: boolean;
+};
+
+const previewKind = async (
+  initialPath: string | undefined,
+  options: ViewOptions,
+): Promise<"run" | "suite" | "experiment" | undefined> => {
+  if (initialPath) {
+    if (isExperimentRun(initialPath)) return "experiment";
+    if (await loadSuiteRun(initialPath)) return "suite";
+    return "run";
+  }
+  if (options.experiments) return "experiment";
+  if (options.suites) return "suite";
+  return undefined;
+};
+
+const webViewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.json) {
+    throw new InvalidArgumentError("--web and --json are mutually exclusive");
+  }
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  const initialPath = runDirArg ? resolve(runDirArg) : undefined;
+  if (initialPath && !(await ensureDirectory("view", initialPath))) return;
+  const initialKind = await previewKind(initialPath, options);
+  const controller = abortOnInterrupt();
+  const preview = await startPreviewServer({
+    runsRoot,
+    ...(options.port !== undefined ? { port: options.port } : {}),
+    ...(initialPath ? { initialPath } : {}),
+    ...(initialKind ? { initialKind } : {}),
+    ...(options.case ? { caseFilter: options.case } : {}),
+    signal: controller.signal,
+  });
+  process.stdout.write(`Lux Review: ${preview.url}\n`);
+  process.stdout.write(`Reading evidence from ${runsRoot}\n`);
+  if (options.open !== false) {
+    try {
+      await openPreviewUrl(preview.url);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      process.stderr.write(`lux view: could not open a browser (${message})\n`);
+    }
+  }
+  await preview.closed;
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+};
+
+export const viewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.web) {
+    await webViewCommand(runDirArg, options);
+    return;
+  }
+  if (options.port !== undefined || options.open === false) {
+    throw new InvalidArgumentError("--port and --no-open require --web");
+  }
+  if (runDirArg) {
+    const path = resolve(runDirArg);
+    if (!(await ensureDirectory("view", path))) return;
+    if (isExperimentRun(path)) {
+      const experiment = await loadExperimentRun(path);
+      if (!experiment) {
+        process.stderr.write(`lux view: invalid experiment at ${path}\n`);
+        failWith(1);
+        return;
+      }
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
+          : `${formatExperimentRun(path, experiment)}\n`,
+      );
+      return;
+    }
+    const suite = await loadSuiteRun(path);
+    if (suite) {
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
+          : `${formatSuiteRunDetail(path, suite)}\n`,
+      );
+      return;
+    }
+    const detail = await loadRunDetail(path);
+    process.stdout.write(
+      options.json ? `${JSON.stringify(detail, null, 2)}\n` : `${formatRunDetail(detail)}\n`,
+    );
+    return;
+  }
+
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  if (options.experiments) {
+    const experiments = await listExperimentRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(experiments, null, 2)}\n`
+        : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
+    );
+    return;
+  }
+  if (options.suites) {
+    const suites = await listSuiteRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(suites, null, 2)}\n`
+        : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
+    );
+    return;
+  }
+  const all = await listRuns(runsRoot);
+  const runs = options.case ? all.filter((run) => run.caseSlug === options.case) : all;
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+    return;
+  }
+  if (!options.case) {
+    process.stdout.write(`suite runs at ${runsRoot}:\n`);
+    process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
+  }
+  process.stdout.write(`runs at ${runsRoot}:\n`);
+  process.stdout.write(`${formatRunsTable(runs)}\n`);
+};
+
+export const parsePreviewPort = (value: string): number => {
+  const port = Number(value);
+  if (!Number.isInteger(port) || port < 0 || port > 65_535) {
+    throw new InvalidArgumentError("port must be an integer from 0 to 65535");
+  }
+  return port;
+};

package/src/core/action-events.ts

@@ -0,0 +1,59 @@
+import { z } from "zod";
+import type { RunEvent } from "./events.js";
+
+export const ActionCategorySchema = z.enum([
+  "query",
+  "mutation",
+  "durable-mutation",
+  "execution",
+  "interaction",
+]);
+export type ActionCategory = z.infer<typeof ActionCategorySchema>;
+
+export const ActionProgressPayloadSchema = z
+  .object({
+    name: z.string(),
+    category: ActionCategorySchema,
+  })
+  .strict();
+export type ActionProgressPayload = z.infer<typeof ActionProgressPayloadSchema>;
+
+export const ActionTimingSchema = z.object({
+  timeToFirstActionMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstMutationMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstDurableMutationMs: z.exactOptional(z.number().nonnegative()),
+});
+export type ActionTiming = z.infer<typeof ActionTimingSchema>;
+
+export const actionTiming = (events: readonly RunEvent[]): ActionTiming => {
+  const startEvent = events.find(
+    (event) => event.type === "lifecycle" && event.stage === "start-driver",
+  );
+  const startedAt = startEvent?.type === "lifecycle" ? startEvent.ts : undefined;
+  if (startedAt === undefined) return {};
+
+  const actions = events.flatMap((event) => {
+    if (event.type !== "progress" || event.kind !== "action") return [];
+    const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+    return payload ? [{ ...payload, ts: event.ts }] : [];
+  });
+  const elapsed = (category: "any" | "mutation" | "durable-mutation") => {
+    const timestamps = actions.flatMap(({ category: candidate, ts }) => {
+      if (category === "any") return [ts];
+      if (category === "mutation") {
+        return candidate === "mutation" || candidate === "durable-mutation" ? [ts] : [];
+      }
+      return candidate === category ? [ts] : [];
+    });
+    return timestamps.length > 0 ? Math.max(0, Math.min(...timestamps) - startedAt) : undefined;
+  };
+
+  const timeToFirstActionMs = elapsed("any");
+  const timeToFirstMutationMs = elapsed("mutation");
+  const timeToFirstDurableMutationMs = elapsed("durable-mutation");
+  return {
+    ...(timeToFirstActionMs === undefined ? {} : { timeToFirstActionMs }),
+    ...(timeToFirstMutationMs === undefined ? {} : { timeToFirstMutationMs }),
+    ...(timeToFirstDurableMutationMs === undefined ? {} : { timeToFirstDurableMutationMs }),
+  };
+};

package/src/core/driver-metrics.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const DriverMetricUnitSchema = z.enum([
+  "number",
+  "count",
+  "milliseconds",
+  "bytes",
+  "tokens",
+  "usd",
+  "ratio",
+  "percent",
+  "score",
+]);
+export type DriverMetricUnit = z.infer<typeof DriverMetricUnitSchema>;
+
+export const DriverMetricDescriptorSchema = z
+  .object({
+    unit: DriverMetricUnitSchema.default("number"),
+    label: z.string().min(1).max(120).optional(),
+    better: z.enum(["lower", "higher", "neutral"]).default("neutral"),
+  })
+  .strict();
+export type DriverMetricDescriptor = z.infer<typeof DriverMetricDescriptorSchema>;
+
+export const DriverMetricSchema = DriverMetricDescriptorSchema.extend({
+  value: z.number().finite(),
+}).strict();
+export type DriverMetric = z.infer<typeof DriverMetricSchema>;
+
+export const DriverMetricsSchema = z.record(z.string().min(1).max(120), DriverMetricSchema);
+export type DriverMetrics = z.infer<typeof DriverMetricsSchema>;
+
+const ReportedDriverMetricSchema = z.union([z.number().finite(), DriverMetricSchema]);
+export const ReportedDriverMetricsSchema = z.record(
+  z.string().min(1).max(120),
+  ReportedDriverMetricSchema,
+);
+export type ReportedDriverMetrics = z.input<typeof ReportedDriverMetricsSchema>;
+
+export const normalizeDriverMetrics = (value: unknown): DriverMetrics => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value).flatMap(([name, metric]) => {
+      if (name.length === 0 || name.length > 120) return [];
+      const parsed = DriverMetricSchema.safeParse(
+        typeof metric === "number" ? { value: metric } : metric,
+      ).data;
+      return parsed ? [[name, parsed]] : [];
+    }),
+  );
+};

package/src/core/events.ts

@@ -69,8 +69,8 @@ export const DoneSchema = z.object({
   exitReason: z.literal("done"),
   /**
    * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
-   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
-   * for domain-specific measurements.
+   * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
+   * preserving room for other domain-specific evidence.
    */
   summary: z.unknown().optional(),
 });

package/src/core/experiment.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
 import { ConfidenceConfigSchema } from "./statistics.js";
 import { TokenUsageSchema } from "./usage.js";
 import { formatZodIssues } from "./zod.js";
@@ -415,6 +416,7 @@ export const CandidateEvaluationSchema = z.object({
   passRate: z.number().min(0).max(1),
   casePassRates: z.record(z.string(), z.number().min(0).max(1)),
   metrics: z.record(z.string(), z.number()),
+  metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
   metricCalls: z.exactOptional(z.number().int().nonnegative()),
   judgeCalls: z.exactOptional(z.number().int().nonnegative()),
   unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),

package/src/core/index.ts

@@ -1,3 +1,4 @@
+export * from "./action-events.js";
 export * from "./annotation.js";
 export * from "./answerer.js";
 export * from "./artifact-evidence.js";
@@ -6,6 +7,7 @@ export * from "./assertion.js";
 export * from "./case.js";
 export * from "./conversation-turn.js";
 export * from "./driver.js";
+export * from "./driver-metrics.js";
 export * from "./emitter.js";
 export * from "./events.js";
 export * from "./exec.js";

package/src/core/run.ts

@@ -1,6 +1,8 @@
 import { z } from "zod";
+import { ActionTimingSchema } from "./action-events.js";
 import { EvalCaseSchema } from "./case.js";
 import { ArtifactSchema } from "./driver.js";
+import { DriverMetricsSchema } from "./driver-metrics.js";
 import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
 import { ToolOutcomeSummarySchema } from "./tool-events.js";
 import { RunUsageSchema } from "./usage.js";
@@ -112,6 +114,8 @@ export const RunMetadataSchema = z.object({
     }),
   ),
   cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
+  /** Driver-defined scalar measurements with display and optimization semantics. */
+  driverMetrics: z.exactOptional(DriverMetricsSchema),
   interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
   /** Legacy raw count of progress events whose kind is `tool-call`. */
   toolCallCount: z.exactOptional(z.number()),
@@ -124,6 +128,8 @@ export const RunMetadataSchema = z.object({
   ),
   /** Wall-clock per phase, derived from the lifecycle event stream. */
   phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
+  /** Driver-reported semantic action latency, measured from `start-driver`. */
+  actionTiming: z.exactOptional(ActionTimingSchema),
   /**
    * Harness/infrastructure failure that invalidates this run as candidate
    * evidence. A driver-reported subject failure intentionally leaves this

package/src/index.ts

@@ -87,6 +87,8 @@ export {
   registerCoreAssertions,
   runSucceededAssertion,
   subagentToolCalls,
+  type TimeToFirstActionUnderConfig,
+  TimeToFirstActionUnderConfigSchema,
   type TokensUnderConfig,
   TokensUnderConfigSchema,
   type ToolCall,
@@ -104,6 +106,7 @@ export {
   ToolResultIsErrorConfigSchema,
   type ToolResultJsonPointerEqualsConfig,
   ToolResultJsonPointerEqualsConfigSchema,
+  timeToFirstActionUnderAssertion,
   tokensUnderAssertion,
   toolCallCountAssertion,
   toolCalledAssertion,
@@ -141,6 +144,15 @@ export {
   rubricAssertion,
 } from "./assertions/rubric/index.js";
 // Authoring and extension contracts.
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
 export {
   defineGraderAnnotations,
   type GraderAnnotation,
@@ -227,6 +239,19 @@ export {
   ResolvedFixturesSchema,
   type StartContext,
 } from "./core/driver.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
 export {
   type Answer,

package/src/orchestrator/compare.ts

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  type ActionTiming,
+  type DriverMetrics,
   emptyRunUsage,
   type GradingReport,
   GradingReportSchema,
@@ -48,6 +50,8 @@ export type RunComparison = {
     exitReason: { a: string; b: string };
     durationMsA: number | null;
     durationMsB: number | null;
+    actionTiming: { a: ActionTiming; b: ActionTiming };
+    driverMetrics: { a: DriverMetrics; b: DriverMetrics };
     interruptCounts: {
       a: { ask: number; approve: number; custom: number };
       b: { ask: number; approve: number; custom: number };
@@ -260,6 +264,14 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
           },
           durationMsA: durationFromMeta(a.runJson),
           durationMsB: durationFromMeta(b.runJson),
+          actionTiming: {
+            a: a.runJson?.actionTiming ?? {},
+            b: b.runJson?.actionTiming ?? {},
+          },
+          driverMetrics: {
+            a: a.runJson?.driverMetrics ?? {},
+            b: b.runJson?.driverMetrics ?? {},
+          },
           interruptCounts: {
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },

package/src/orchestrator/experiment-evaluation.ts

@@ -101,6 +101,23 @@ const evaluationMetrics = (
   ),
 });
 
+const evaluationMetricDefinitions = (summary: SuiteSummary) =>
+  Object.fromEntries(
+    Object.entries(summary.metrics).flatMap(([name, metric]) => {
+      if (!metric.unit && !metric.label && !metric.better) return [];
+      return [
+        [
+          name,
+          {
+            unit: metric.unit ?? "number",
+            better: metric.better ?? "neutral",
+            ...(metric.label ? { label: metric.label } : {}),
+          },
+        ],
+      ];
+    }),
+  );
+
 const summaryUsage = (summary: SuiteSummary) =>
   summary.samples.reduce(
     (usage, sample) =>
@@ -277,6 +294,7 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
+    metricDefinitions: evaluationMetricDefinitions(summary),
     samples: evaluationSamples(summary),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
@@ -352,6 +370,10 @@ const aggregateAgentUnits = (
         .map(([, values]) => average(values)),
     ]),
   );
+  const metricDefinitions = Object.assign(
+    {},
+    ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
+  );
   return {
     candidateId: candidate.id,
     profileId,
@@ -367,6 +389,7 @@ const aggregateAgentUnits = (
         name === "subject.bytes" ? candidate.sourceBytes : average(values),
       ]),
     ),
+    metricDefinitions,
     samples,
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),

package/src/orchestrator/index.ts

@@ -122,6 +122,7 @@ export {
   formatRunsTable,
   type ListedRun,
   listRuns,
+  loadRunArtifacts,
   loadRunDetail,
   type RunDetail,
 } from "./list-runs.js";