@prismatic-io/lux 0.0.1 → 0.0.2-preview.11

package/src/core/agent-activity.ts

@@ -0,0 +1,236 @@
+import { z } from "zod";
+import { ActionProgressPayloadSchema } from "./action-events.js";
+import { AgentCompletedPayloadSchema, AgentSpawnPayloadSchema } from "./agent-events.js";
+import type { AgentAttribution, ReadyState, RunEvent } from "./events.js";
+import { ToolCallPayloadSchema, ToolResultPayloadSchema } from "./tool-events.js";
+import { TokenUsageSchema } from "./usage.js";
+
+export const AgentActivitySchema = z.object({
+  agent: z.object({
+    id: z.string(),
+    role: z.enum(["main", "subagent"]),
+    parentId: z.exactOptional(z.string()),
+    spawnToolUseId: z.exactOptional(z.string()),
+    name: z.exactOptional(z.string()),
+    model: z.exactOptional(z.string()),
+  }),
+  startedAt: z.number(),
+  endedAt: z.number(),
+  durationMs: z.number().nonnegative(),
+  status: z.enum(["completed", "failed", "cancelled", "unknown"]),
+  eventCount: z.number().int().nonnegative(),
+  usage: TokenUsageSchema.extend({ total: z.number().int().nonnegative() }),
+  toolCalls: z.object({
+    total: z.number().int().nonnegative(),
+    succeeded: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    unresolved: z.number().int().nonnegative(),
+  }),
+  actions: z.object({
+    total: z.number().int().nonnegative(),
+    mutations: z.number().int().nonnegative(),
+    durableMutations: z.number().int().nonnegative(),
+    timeToFirstActionMs: z.exactOptional(z.number().nonnegative()),
+    timeToFirstMutationMs: z.exactOptional(z.number().nonnegative()),
+    timeToFirstDurableMutationMs: z.exactOptional(z.number().nonnegative()),
+  }),
+});
+export type AgentActivity = z.infer<typeof AgentActivitySchema>;
+
+const legacyAttribution = (
+  event: Extract<RunEvent, { type: "progress" }>,
+  ready: ReadyState,
+): AgentAttribution => {
+  const payload = event.payload;
+  const parentToolUseId =
+    typeof payload === "object" && payload !== null && "parentToolUseId" in payload
+      ? payload.parentToolUseId
+      : undefined;
+  const spawnToolUseId =
+    typeof parentToolUseId === "string" && parentToolUseId.length > 0 ? parentToolUseId : undefined;
+  return spawnToolUseId
+    ? {
+        id: `subagent:${spawnToolUseId}`,
+        role: "subagent" as const,
+        parentId: ready.agentId,
+        spawnToolUseId,
+      }
+    : { id: ready.agentId, role: "main" as const, ...(ready.model ? { model: ready.model } : {}) };
+};
+
+type MutableActivity = AgentActivity & { calls: Map<string, "pending" | "succeeded" | "failed"> };
+type ProgressEvent = Extract<RunEvent, { type: "progress" }>;
+type SpawnInfo = { ts: number; name?: string; model?: string };
+
+const initialActivity = (agent: AgentAttribution, startedAt: number): MutableActivity => ({
+  agent,
+  startedAt,
+  endedAt: startedAt,
+  durationMs: 0,
+  status: "unknown",
+  eventCount: 0,
+  usage: { input: 0, output: 0, cacheRead: 0, cacheCreation: 0, total: 0 },
+  toolCalls: { total: 0, succeeded: 0, failed: 0, unresolved: 0 },
+  actions: { total: 0, mutations: 0, durableMutations: 0 },
+  calls: new Map(),
+});
+
+const collectSpawnInfo = (events: RunEvent[]): Map<string, SpawnInfo> => {
+  const spawns = new Map<string, SpawnInfo>();
+  for (const event of events) {
+    if (event.type !== "progress" || event.kind !== "tool-call") continue;
+    const call = ToolCallPayloadSchema.safeParse(event.payload).data;
+    if (!call) continue;
+    const input = z
+      .object({
+        name: z.string().min(1).optional(),
+        subagent_type: z.string().min(1).optional(),
+        agent: z.string().min(1).optional(),
+        role: z.string().min(1).optional(),
+        model: z.string().min(1).optional(),
+      })
+      .loose()
+      .safeParse(call.input).data;
+    const name = input?.name ?? input?.subagent_type ?? input?.agent ?? input?.role;
+    spawns.set(call.toolUseId, {
+      ts: event.ts,
+      ...(name ? { name } : {}),
+      ...(input?.model ? { model: input.model } : {}),
+    });
+  }
+  return spawns;
+};
+
+const applyLifecycle = (
+  event: ProgressEvent,
+  activities: Map<string, MutableActivity>,
+): boolean => {
+  if (event.kind === "agent-spawn") {
+    const spawn = AgentSpawnPayloadSchema.safeParse(event.payload).data;
+    if (spawn && !activities.has(spawn.agent.id)) {
+      activities.set(spawn.agent.id, initialActivity(spawn.agent, event.ts));
+    }
+    return true;
+  }
+  if (event.kind !== "agent-completed") return false;
+  const completed = AgentCompletedPayloadSchema.safeParse(event.payload).data;
+  const activity = completed ? activities.get(completed.agentId) : undefined;
+  if (activity && completed) {
+    activity.endedAt = Math.max(activity.endedAt, event.ts);
+    activity.durationMs = Math.max(0, activity.endedAt - activity.startedAt);
+    activity.status = completed.status;
+  }
+  return true;
+};
+
+const activityFor = (
+  event: ProgressEvent,
+  ready: ReadyState,
+  driverStartedAt: number,
+  spawns: Map<string, SpawnInfo>,
+  activities: Map<string, MutableActivity>,
+): MutableActivity => {
+  const agent = event.agent ?? legacyAttribution(event, ready);
+  const existing = activities.get(agent.id);
+  if (existing) return existing;
+  const spawn = agent.spawnToolUseId ? spawns.get(agent.spawnToolUseId) : undefined;
+  const name = agent.name ?? spawn?.name;
+  const model = agent.model ?? spawn?.model;
+  const attributedAgent = {
+    ...agent,
+    ...(name ? { name } : {}),
+    ...(model ? { model } : {}),
+  };
+  const startedAt =
+    agent.role === "subagent" && agent.spawnToolUseId
+      ? (spawn?.ts ?? event.ts)
+      : driverStartedAt || event.ts;
+  return activities.get(agent.id) ?? initialActivity(attributedAgent, startedAt);
+};
+
+const addAction = (event: ProgressEvent, activity: MutableActivity): void => {
+  const action = ActionProgressPayloadSchema.safeParse(event.payload).data;
+  if (!action) return;
+  const elapsed = Math.max(0, event.ts - activity.startedAt);
+  activity.actions.total++;
+  activity.actions.timeToFirstActionMs ??= elapsed;
+  if (action.category === "mutation" || action.category === "durable-mutation") {
+    activity.actions.mutations++;
+    activity.actions.timeToFirstMutationMs ??= elapsed;
+  }
+  if (action.category === "durable-mutation") {
+    activity.actions.durableMutations++;
+    activity.actions.timeToFirstDurableMutationMs ??= elapsed;
+  }
+};
+
+const addUsage = (event: ProgressEvent, activity: MutableActivity): void => {
+  const usage = TokenUsageSchema.safeParse(event.payload).data;
+  if (!usage) return;
+  activity.usage.input += usage.input;
+  activity.usage.output += usage.output;
+  activity.usage.cacheRead += usage.cacheRead;
+  activity.usage.cacheCreation += usage.cacheCreation;
+  activity.usage.total += usage.input + usage.output + usage.cacheRead + usage.cacheCreation;
+};
+
+const addWork = (event: ProgressEvent, activity: MutableActivity): void => {
+  activity.eventCount++;
+  activity.endedAt = Math.max(activity.endedAt, event.ts);
+  activity.durationMs = Math.max(0, activity.endedAt - activity.startedAt);
+  if (event.kind === "tool-call") {
+    const call = ToolCallPayloadSchema.safeParse(event.payload).data;
+    if (call && !activity.calls.has(call.toolUseId)) {
+      activity.calls.set(call.toolUseId, "pending");
+      activity.toolCalls.total++;
+    }
+    return;
+  }
+  if (event.kind === "tool-result") {
+    const result = ToolResultPayloadSchema.safeParse(event.payload).data;
+    if (result && activity.calls.has(result.toolUseId)) {
+      activity.calls.set(result.toolUseId, result.isError ? "failed" : "succeeded");
+    }
+    return;
+  }
+  if (event.kind === "action") addAction(event, activity);
+  if (event.kind === "usage") addUsage(event, activity);
+};
+
+export const summarizeAgentActivity = (events: RunEvent[], ready: ReadyState): AgentActivity[] => {
+  const driverStart = events.find(
+    (event): event is Extract<RunEvent, { type: "lifecycle" }> =>
+      event.type === "lifecycle" && event.stage === "start-driver",
+  );
+  const driverStartedAt = driverStart?.ts ?? 0;
+  const spawns = collectSpawnInfo(events);
+  const activities = new Map<string, MutableActivity>();
+  for (const event of events) {
+    if (event.type !== "progress") continue;
+    if (applyLifecycle(event, activities)) continue;
+    const activity = activityFor(event, ready, driverStartedAt, spawns, activities);
+    addWork(event, activity);
+    activities.set(activity.agent.id, activity);
+  }
+
+  if (!activities.has(ready.agentId)) {
+    activities.set(
+      ready.agentId,
+      initialActivity(
+        {
+          id: ready.agentId,
+          role: "main",
+          ...(ready.model ? { model: ready.model } : {}),
+        },
+        driverStartedAt,
+      ),
+    );
+  }
+  return [...activities.values()]
+    .map(({ calls, ...activity }) => {
+      for (const outcome of calls.values())
+        activity.toolCalls[outcome === "pending" ? "unresolved" : outcome]++;
+      return AgentActivitySchema.parse(activity);
+    })
+    .sort((a, b) => a.startedAt - b.startedAt || a.agent.id.localeCompare(b.agent.id));
+};

package/src/core/agent-events.ts

@@ -0,0 +1,17 @@
+import { z } from "zod";
+import { AgentAttributionSchema } from "./events.js";
+
+export const AGENT_SPAWN_KIND = "agent-spawn" as const;
+export const AGENT_COMPLETED_KIND = "agent-completed" as const;
+
+export const AgentSpawnPayloadSchema = z.object({
+  agent: AgentAttributionSchema.extend({ role: z.literal("subagent") }),
+  prompt: z.exactOptional(z.string()),
+});
+export type AgentSpawnPayload = z.infer<typeof AgentSpawnPayloadSchema>;
+
+export const AgentCompletedPayloadSchema = z.object({
+  agentId: z.string().min(1),
+  status: z.enum(["completed", "failed", "cancelled", "unknown"]),
+});
+export type AgentCompletedPayload = z.infer<typeof AgentCompletedPayloadSchema>;

package/src/core/artifact-evidence.ts

@@ -7,12 +7,15 @@ import type { Run } from "./run.js";
 const DEFAULT_ARTIFACT_READ_BYTES = 5 * 1024 * 1024;
 const MAX_ARTIFACT_READ_BYTES = 10 * 1024 * 1024;
 
+export type RunArtifactIssueReason = "absent" | "unreadable";
+
 export type RunArtifactInspection = {
   target: string;
   root: string;
   artifact?: Artifact;
   stats?: Stats;
   issue?: string;
+  reason?: RunArtifactIssueReason;
 };
 
 type RunArtifactEvidence = {
@@ -82,6 +85,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: runArtifactsRoot(run),
       issue: `${authoredPath} contains a null byte`,
+      reason: "unreadable",
     };
   }
   const matching = run.artifacts.filter((candidate) => candidate.path === authoredPath);
@@ -91,6 +95,7 @@ export const inspectRunArtifact = async (
       target: resolve(root, authoredPath),
       root,
       issue: `${authoredPath} was not captured in the frozen artifact index`,
+      reason: "absent",
     };
   }
   if (matching.some((candidate) => candidate.root !== undefined && !isAbsolute(candidate.root))) {
@@ -98,6 +103,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} declares a non-absolute artifact root`,
+      reason: "unreadable",
     };
   }
   const artifact = matching[0];
@@ -108,6 +114,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} is ambiguous across multiple artifact roots`,
+      reason: "unreadable",
     };
   }
   // A non-default root is trusted only for an exact artifact-index entry.
@@ -116,7 +123,12 @@ export const inspectRunArtifact = async (
   const root = matchingRoots[0] ?? defaultArtifactsRoot(run);
   const target = resolve(root, authoredPath);
   if (pathEscapesRoot(root, target)) {
-    return { target, root, issue: `${authoredPath} escapes the captured artifact root` };
+    return {
+      target,
+      root,
+      issue: `${authoredPath} escapes the captured artifact root`,
+      reason: "unreadable",
+    };
   }
 
   const path = relative(root, target);
@@ -134,7 +146,13 @@ export const inspectRunArtifact = async (
     } catch (error) {
       const code = (error as NodeJS.ErrnoException).code;
       if (code === "ENOENT" || code === "ENOTDIR") {
-        return { target, root, ...identified, issue: `${authoredPath} not found` };
+        return {
+          target,
+          root,
+          ...identified,
+          issue: `${authoredPath} not found`,
+          reason: "absent",
+        };
       }
       throw error;
     }
@@ -144,16 +162,17 @@ export const inspectRunArtifact = async (
         root,
         ...identified,
         issue: `${authoredPath} resolves through a symlink`,
+        reason: "unreadable",
       };
     }
     if (index < components.length - 1 && !stats.isDirectory()) {
-      return { target, root, ...identified, issue: `${authoredPath} not found` };
+      return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
     }
     if (index === components.length - 1) {
       return { target, root, ...identified, stats };
     }
   }
-  return { target, root, ...identified, issue: `${authoredPath} not found` };
+  return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
 };
 
 export type ReadRunArtifactTextOptions = {
@@ -247,3 +266,27 @@ export const readRunArtifactText = async (
 ): Promise<string> => {
   return (await readRunArtifactBytes(run, authoredPath, options)).toString("utf8");
 };
+
+/**
+ * Read optional artifact evidence. Returns null when the artifact was never
+ * captured, and still throws when one exists but is unsafe to consume, so a
+ * traversal or symlink violation is never mistaken for missing evidence.
+ */
+export const readRunArtifactTextIfPresent = async (
+  run: RunArtifactEvidence,
+  authoredPath: string,
+  options: ReadRunArtifactTextOptions = {},
+): Promise<string | null> => {
+  const inspected = await inspectRunArtifact(run, authoredPath);
+  if (inspected.reason === "absent") return null;
+  if (!inspected.stats?.isFile()) {
+    throw new Error(
+      inspected.issue ??
+        (inspected.stats ? `${authoredPath} is not a file` : `${authoredPath} not found`),
+    );
+  }
+  const maxBytes = options.maxBytes ?? DEFAULT_ARTIFACT_READ_BYTES;
+  return (
+    await readInspectedFileBytes(inspected.target, inspected.stats, maxBytes, authoredPath)
+  ).toString("utf8");
+};

package/src/core/assertion.ts

@@ -42,34 +42,23 @@ export interface AssertionImpl<C extends Assertion = Assertion> {
  * `AssertionImpl`, plugins must describe their case configuration so Lux can
  * reject invalid plans before starting a subject run.
  */
-export type AssertionPlugin<C extends Assertion = Assertion> = AssertionImpl<C> & {
+export type AssertionPlugin<
+  C extends Assertion = Assertion,
+  Input extends Assertion = C,
+> = AssertionImpl<C> & {
   configSchema: z.ZodObject & z.ZodType<C>;
+  /** Type-only input accepted while authoring before schema defaults apply. */
+  readonly __input?: Input;
   /** Author-maintained identity for evaluator/schema behavior. */
   pluginVersion: string;
 };
 
 type SchemaAssertion<S extends z.ZodObject> = z.output<S> extends Assertion ? z.output<S> : never;
-
-type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
-
-type SingleStringLiteral<T> = T extends string
-  ? string extends T
-    ? never
-    : true extends IsUnion<T>
-      ? never
-      : T
-  : never;
+type SchemaAssertionInput<S extends z.ZodObject> =
+  z.input<S> extends Assertion ? z.input<S> : never;
 
 type LiteralAssertionType<S extends z.ZodObject> =
-  SchemaAssertion<S> extends { type: infer OutputType }
-    ? z.input<S> extends { type: infer InputType }
-      ? SingleStringLiteral<OutputType> extends infer Literal extends string
-        ? [InputType, Literal] extends [Literal, InputType]
-          ? Literal
-          : never
-        : never
-      : never
-    : never;
+  S["shape"]["type"] extends z.ZodLiteral<infer Literal extends string> ? Literal : never;
 
 export type AssertionPluginDefinition<S extends z.ZodObject> = {
   type: LiteralAssertionType<S>;
@@ -111,12 +100,34 @@ export const assertionPluginIssue = (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
 export const defineAssertion = <S extends z.ZodObject>(
   definition: AssertionPluginDefinition<S>,
-): AssertionPlugin<SchemaAssertion<S>> => {
+): AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>> => {
   const issue = assertionPluginIssue(definition);
   if (issue) throw new TypeError(issue);
-  return definition as unknown as AssertionPlugin<SchemaAssertion<S>>;
+  return definition as unknown as AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>>;
+};
+
+export const defineAssertionRegistry = <const Plugins extends readonly AssertionPlugin[]>(
+  ...plugins: Plugins
+): Plugins => {
+  const types = new Set<string>();
+  for (const plugin of plugins) {
+    const issue = assertionPluginIssue(plugin);
+    if (issue) throw new TypeError(issue);
+    if (types.has(plugin.type)) {
+      throw new TypeError(`assertion registry has duplicate type '${plugin.type}'`);
+    }
+    types.add(plugin.type);
+  }
+  return plugins;
 };
 
 export const GradingAssertionEntrySchema = z.object({
@@ -137,11 +148,19 @@ export const GradingAssertionEntrySchema = z.object({
 });
 export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
 
+const SkippedGradingAssertionSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+  reason: z.literal("judge-disabled"),
+});
+
 export const GradingReportSchema = z.object({
   schemaVersion: z.literal(1),
   casePassed: z.boolean().nullable(),
   caseScore: z.number().nullable(),
   assertions: z.array(GradingAssertionEntrySchema),
+  skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
   /**
    * Present when the subject run or grading did not complete. Assertions may
    * still contain partial diagnostics, but the report is not a clean verdict.

package/src/core/case-authoring-state.ts

@@ -0,0 +1,65 @@
+import type { AssertionPlugin } from "./assertion.js";
+import type { EvalCase } from "./case.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
+
+export type EvalCaseExtensions = {
+  assertions: readonly AssertionPlugin[];
+  fixtures: readonly LifecycleHooksPlugin[];
+};
+
+const CASE_EXTENSIONS_KEY = Symbol.for("@prismatic-io/lux/eval-case-extensions");
+const extensionState = globalThis as typeof globalThis & { [key: symbol]: unknown };
+const sharedCaseExtensions = extensionState[CASE_EXTENSIONS_KEY];
+// A project-local package and the CLI can load separate Lux module instances.
+const caseExtensions =
+  sharedCaseExtensions instanceof WeakMap
+    ? (sharedCaseExtensions as WeakMap<object, EvalCaseExtensions>)
+    : new WeakMap<object, EvalCaseExtensions>();
+extensionState[CASE_EXTENSIONS_KEY] = caseExtensions;
+
+const uniqueBy = <T>(values: readonly T[], keyOf: (value: T) => string, kind: string): T[] => {
+  const unique = new Map<string, T>();
+  for (const value of values) {
+    const key = keyOf(value);
+    const existing = unique.get(key);
+    if (existing && existing !== value) {
+      throw new TypeError(`case has conflicting ${kind} definitions for '${key}'`);
+    }
+    unique.set(key, value);
+  }
+  return [...unique.values()];
+};
+
+export const collectEvalCaseExtensions = (
+  input: EvalCase,
+  assertions: readonly AssertionPlugin[] = [],
+  fixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseExtensions => {
+  const availableAssertions = uniqueBy(assertions, (plugin) => plugin.type, "assertion");
+  const byType = new Map(availableAssertions.map((plugin) => [plugin.type, plugin]));
+  return {
+    assertions: uniqueBy(
+      (input.assertions ?? []).flatMap((assertion) => {
+        const plugin = byType.get(assertion.type);
+        return plugin ? [plugin] : [];
+      }),
+      (plugin) => plugin.type,
+      "assertion",
+    ),
+    fixtures: uniqueBy(fixtures, (fixture) => fixture.name, "fixture"),
+  };
+};
+
+export const setEvalCaseExtensions = (evalCase: EvalCase, extensions: EvalCaseExtensions): void => {
+  if (extensions.assertions.length > 0 || extensions.fixtures.length > 0) {
+    caseExtensions.set(evalCase, extensions);
+  }
+};
+
+export const evalCaseExtensions = (evalCase: EvalCase): EvalCaseExtensions =>
+  caseExtensions.get(evalCase) ?? { assertions: [], fixtures: [] };
+
+export const transferEvalCaseExtensions = (source: EvalCase, target: EvalCase): void => {
+  const extensions = caseExtensions.get(source);
+  if (extensions) caseExtensions.set(target, extensions);
+};

package/src/core/case.ts

@@ -1,4 +1,7 @@
 import { z } from "zod";
+import type { AssertionPlugin } from "./assertion.js";
+import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
 import { formatZodIssues } from "./zod.js";
 
 export const FixturesRefSchema = z.union([
@@ -67,14 +70,62 @@ export const EvalCaseSchema = EvalCaseObjectSchema
   });
 export type EvalCase = z.infer<typeof EvalCaseSchema>;
 
-export const defineEvalCase = (input: EvalCase): EvalCase => {
-  const parsed = EvalCaseSchema.safeParse(input);
-  if (!parsed.success) {
-    throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
-  }
-  return parsed.data;
+type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
+type AssertionInputOf<Plugin> =
+  Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
+
+export type EvalCaseInput<AllowedAssertion extends Assertion = Assertion> = Omit<
+  EvalCase,
+  "assertions"
+> & {
+  assertions?: AllowedAssertion[];
+};
+
+export type EvalCaseAuthoringOptions<
+  Plugins extends readonly AssertionPlugin[] = readonly AssertionPlugin[],
+> = {
+  assertions?: Plugins;
+  fixtures?: readonly LifecycleHooksPlugin[];
+};
+
+export type EvalCaseDefinition<AllowedAssertion extends Assertion = Assertion> = {
+  (input: EvalCaseInput<AllowedAssertion>): EvalCase;
+  /**
+   * Create a case definition with selectively activated lifecycle fixtures.
+   * Further calls to `extend` append fixtures in setup order.
+   */
+  extend<const Plugins extends readonly AssertionPlugin[] = readonly []>(
+    options: EvalCaseAuthoringOptions<Plugins>,
+  ): EvalCaseDefinition<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
 };
 
+export const createEvalCaseDefinition = <AllowedAssertion extends Assertion = Assertion>(
+  inheritedAssertions: readonly AssertionPlugin[] = [],
+  inheritedFixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseDefinition<AllowedAssertion> => {
+  const define = ((input: EvalCaseInput<AllowedAssertion>): EvalCase => {
+    const parsed = EvalCaseSchema.safeParse(input);
+    if (!parsed.success) {
+      throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
+    }
+    const extensions = collectEvalCaseExtensions(
+      parsed.data,
+      inheritedAssertions,
+      inheritedFixtures,
+    );
+    setEvalCaseExtensions(parsed.data, extensions);
+    return parsed.data;
+  }) as EvalCaseDefinition<AllowedAssertion>;
+  define.extend = (({ assertions = [], fixtures = [] }: EvalCaseAuthoringOptions) =>
+    createEvalCaseDefinition(
+      [...inheritedAssertions, ...assertions],
+      [...inheritedFixtures, ...fixtures],
+    )) as EvalCaseDefinition<AllowedAssertion>["extend"];
+  return define;
+};
+
+export const defineEvalCase = createEvalCaseDefinition();
+
 const PersistedFixturesRefSchema = z.union([
   z.string(),
   z.looseObject({ kind: z.literal("dir"), path: z.string() }),

package/src/core/driver-metrics.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const DriverMetricUnitSchema = z.enum([
+  "number",
+  "count",
+  "milliseconds",
+  "bytes",
+  "tokens",
+  "usd",
+  "ratio",
+  "percent",
+  "score",
+]);
+export type DriverMetricUnit = z.infer<typeof DriverMetricUnitSchema>;
+
+export const DriverMetricDescriptorSchema = z
+  .object({
+    unit: DriverMetricUnitSchema.default("number"),
+    label: z.string().min(1).max(120).optional(),
+    better: z.enum(["lower", "higher", "neutral"]).default("neutral"),
+  })
+  .strict();
+export type DriverMetricDescriptor = z.infer<typeof DriverMetricDescriptorSchema>;
+
+export const DriverMetricSchema = DriverMetricDescriptorSchema.extend({
+  value: z.number().finite(),
+}).strict();
+export type DriverMetric = z.infer<typeof DriverMetricSchema>;
+
+export const DriverMetricsSchema = z.record(z.string().min(1).max(120), DriverMetricSchema);
+export type DriverMetrics = z.infer<typeof DriverMetricsSchema>;
+
+const ReportedDriverMetricSchema = z.union([z.number().finite(), DriverMetricSchema]);
+export const ReportedDriverMetricsSchema = z.record(
+  z.string().min(1).max(120),
+  ReportedDriverMetricSchema,
+);
+export type ReportedDriverMetrics = z.input<typeof ReportedDriverMetricsSchema>;
+
+export const normalizeDriverMetrics = (value: unknown): DriverMetrics => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value).flatMap(([name, metric]) => {
+      if (name.length === 0 || name.length > 120) return [];
+      const parsed = DriverMetricSchema.safeParse(
+        typeof metric === "number" ? { value: metric } : metric,
+      ).data;
+      return parsed ? [[name, parsed]] : [];
+    }),
+  );
+};

package/src/core/driver.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import type { EventHandler } from "./emitter.js";
 import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
+import type { FixtureValueBinding } from "./lifecycle-hooks.js";
 
 export const ArtifactSchema = z.object({
   path: z.string(),
@@ -28,6 +29,8 @@ export type StartContext = {
   caseId?: string;
   prompt: string;
   fixtures?: ResolvedFixtures | undefined;
+  /** Authored fixture values selected for this case. */
+  fixtureValues?: readonly FixtureValueBinding[] | undefined;
   /** Root containing persisted runs. Drivers may use it as an isolation boundary. */
   runsRoot?: string | undefined;
   runDir: string;