@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.7

package/src/core/agent-activity.ts

@@ -0,0 +1,236 @@
+import { z } from "zod";
+import { ActionProgressPayloadSchema } from "./action-events.js";
+import { AgentCompletedPayloadSchema, AgentSpawnPayloadSchema } from "./agent-events.js";
+import type { AgentAttribution, ReadyState, RunEvent } from "./events.js";
+import { ToolCallPayloadSchema, ToolResultPayloadSchema } from "./tool-events.js";
+import { TokenUsageSchema } from "./usage.js";
+
+export const AgentActivitySchema = z.object({
+  agent: z.object({
+    id: z.string(),
+    role: z.enum(["main", "subagent"]),
+    parentId: z.exactOptional(z.string()),
+    spawnToolUseId: z.exactOptional(z.string()),
+    name: z.exactOptional(z.string()),
+    model: z.exactOptional(z.string()),
+  }),
+  startedAt: z.number(),
+  endedAt: z.number(),
+  durationMs: z.number().nonnegative(),
+  status: z.enum(["completed", "failed", "cancelled", "unknown"]),
+  eventCount: z.number().int().nonnegative(),
+  usage: TokenUsageSchema.extend({ total: z.number().int().nonnegative() }),
+  toolCalls: z.object({
+    total: z.number().int().nonnegative(),
+    succeeded: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    unresolved: z.number().int().nonnegative(),
+  }),
+  actions: z.object({
+    total: z.number().int().nonnegative(),
+    mutations: z.number().int().nonnegative(),
+    durableMutations: z.number().int().nonnegative(),
+    timeToFirstActionMs: z.exactOptional(z.number().nonnegative()),
+    timeToFirstMutationMs: z.exactOptional(z.number().nonnegative()),
+    timeToFirstDurableMutationMs: z.exactOptional(z.number().nonnegative()),
+  }),
+});
+export type AgentActivity = z.infer<typeof AgentActivitySchema>;
+
+const legacyAttribution = (
+  event: Extract<RunEvent, { type: "progress" }>,
+  ready: ReadyState,
+): AgentAttribution => {
+  const payload = event.payload;
+  const parentToolUseId =
+    typeof payload === "object" && payload !== null && "parentToolUseId" in payload
+      ? payload.parentToolUseId
+      : undefined;
+  const spawnToolUseId =
+    typeof parentToolUseId === "string" && parentToolUseId.length > 0 ? parentToolUseId : undefined;
+  return spawnToolUseId
+    ? {
+        id: `subagent:${spawnToolUseId}`,
+        role: "subagent" as const,
+        parentId: ready.agentId,
+        spawnToolUseId,
+      }
+    : { id: ready.agentId, role: "main" as const, ...(ready.model ? { model: ready.model } : {}) };
+};
+
+type MutableActivity = AgentActivity & { calls: Map<string, "pending" | "succeeded" | "failed"> };
+type ProgressEvent = Extract<RunEvent, { type: "progress" }>;
+type SpawnInfo = { ts: number; name?: string; model?: string };
+
+const initialActivity = (agent: AgentAttribution, startedAt: number): MutableActivity => ({
+  agent,
+  startedAt,
+  endedAt: startedAt,
+  durationMs: 0,
+  status: "unknown",
+  eventCount: 0,
+  usage: { input: 0, output: 0, cacheRead: 0, cacheCreation: 0, total: 0 },
+  toolCalls: { total: 0, succeeded: 0, failed: 0, unresolved: 0 },
+  actions: { total: 0, mutations: 0, durableMutations: 0 },
+  calls: new Map(),
+});
+
+const collectSpawnInfo = (events: RunEvent[]): Map<string, SpawnInfo> => {
+  const spawns = new Map<string, SpawnInfo>();
+  for (const event of events) {
+    if (event.type !== "progress" || event.kind !== "tool-call") continue;
+    const call = ToolCallPayloadSchema.safeParse(event.payload).data;
+    if (!call) continue;
+    const input = z
+      .object({
+        name: z.string().min(1).optional(),
+        subagent_type: z.string().min(1).optional(),
+        agent: z.string().min(1).optional(),
+        role: z.string().min(1).optional(),
+        model: z.string().min(1).optional(),
+      })
+      .loose()
+      .safeParse(call.input).data;
+    const name = input?.name ?? input?.subagent_type ?? input?.agent ?? input?.role;
+    spawns.set(call.toolUseId, {
+      ts: event.ts,
+      ...(name ? { name } : {}),
+      ...(input?.model ? { model: input.model } : {}),
+    });
+  }
+  return spawns;
+};
+
+const applyLifecycle = (
+  event: ProgressEvent,
+  activities: Map<string, MutableActivity>,
+): boolean => {
+  if (event.kind === "agent-spawn") {
+    const spawn = AgentSpawnPayloadSchema.safeParse(event.payload).data;
+    if (spawn && !activities.has(spawn.agent.id)) {
+      activities.set(spawn.agent.id, initialActivity(spawn.agent, event.ts));
+    }
+    return true;
+  }
+  if (event.kind !== "agent-completed") return false;
+  const completed = AgentCompletedPayloadSchema.safeParse(event.payload).data;
+  const activity = completed ? activities.get(completed.agentId) : undefined;
+  if (activity && completed) {
+    activity.endedAt = Math.max(activity.endedAt, event.ts);
+    activity.durationMs = Math.max(0, activity.endedAt - activity.startedAt);
+    activity.status = completed.status;
+  }
+  return true;
+};
+
+const activityFor = (
+  event: ProgressEvent,
+  ready: ReadyState,
+  driverStartedAt: number,
+  spawns: Map<string, SpawnInfo>,
+  activities: Map<string, MutableActivity>,
+): MutableActivity => {
+  const agent = event.agent ?? legacyAttribution(event, ready);
+  const existing = activities.get(agent.id);
+  if (existing) return existing;
+  const spawn = agent.spawnToolUseId ? spawns.get(agent.spawnToolUseId) : undefined;
+  const name = agent.name ?? spawn?.name;
+  const model = agent.model ?? spawn?.model;
+  const attributedAgent = {
+    ...agent,
+    ...(name ? { name } : {}),
+    ...(model ? { model } : {}),
+  };
+  const startedAt =
+    agent.role === "subagent" && agent.spawnToolUseId
+      ? (spawn?.ts ?? event.ts)
+      : driverStartedAt || event.ts;
+  return activities.get(agent.id) ?? initialActivity(attributedAgent, startedAt);
+};
+
+const addAction = (event: ProgressEvent, activity: MutableActivity): void => {
+  const action = ActionProgressPayloadSchema.safeParse(event.payload).data;
+  if (!action) return;
+  const elapsed = Math.max(0, event.ts - activity.startedAt);
+  activity.actions.total++;
+  activity.actions.timeToFirstActionMs ??= elapsed;
+  if (action.category === "mutation" || action.category === "durable-mutation") {
+    activity.actions.mutations++;
+    activity.actions.timeToFirstMutationMs ??= elapsed;
+  }
+  if (action.category === "durable-mutation") {
+    activity.actions.durableMutations++;
+    activity.actions.timeToFirstDurableMutationMs ??= elapsed;
+  }
+};
+
+const addUsage = (event: ProgressEvent, activity: MutableActivity): void => {
+  const usage = TokenUsageSchema.safeParse(event.payload).data;
+  if (!usage) return;
+  activity.usage.input += usage.input;
+  activity.usage.output += usage.output;
+  activity.usage.cacheRead += usage.cacheRead;
+  activity.usage.cacheCreation += usage.cacheCreation;
+  activity.usage.total += usage.input + usage.output + usage.cacheRead + usage.cacheCreation;
+};
+
+const addWork = (event: ProgressEvent, activity: MutableActivity): void => {
+  activity.eventCount++;
+  activity.endedAt = Math.max(activity.endedAt, event.ts);
+  activity.durationMs = Math.max(0, activity.endedAt - activity.startedAt);
+  if (event.kind === "tool-call") {
+    const call = ToolCallPayloadSchema.safeParse(event.payload).data;
+    if (call && !activity.calls.has(call.toolUseId)) {
+      activity.calls.set(call.toolUseId, "pending");
+      activity.toolCalls.total++;
+    }
+    return;
+  }
+  if (event.kind === "tool-result") {
+    const result = ToolResultPayloadSchema.safeParse(event.payload).data;
+    if (result && activity.calls.has(result.toolUseId)) {
+      activity.calls.set(result.toolUseId, result.isError ? "failed" : "succeeded");
+    }
+    return;
+  }
+  if (event.kind === "action") addAction(event, activity);
+  if (event.kind === "usage") addUsage(event, activity);
+};
+
+export const summarizeAgentActivity = (events: RunEvent[], ready: ReadyState): AgentActivity[] => {
+  const driverStart = events.find(
+    (event): event is Extract<RunEvent, { type: "lifecycle" }> =>
+      event.type === "lifecycle" && event.stage === "start-driver",
+  );
+  const driverStartedAt = driverStart?.ts ?? 0;
+  const spawns = collectSpawnInfo(events);
+  const activities = new Map<string, MutableActivity>();
+  for (const event of events) {
+    if (event.type !== "progress") continue;
+    if (applyLifecycle(event, activities)) continue;
+    const activity = activityFor(event, ready, driverStartedAt, spawns, activities);
+    addWork(event, activity);
+    activities.set(activity.agent.id, activity);
+  }
+
+  if (!activities.has(ready.agentId)) {
+    activities.set(
+      ready.agentId,
+      initialActivity(
+        {
+          id: ready.agentId,
+          role: "main",
+          ...(ready.model ? { model: ready.model } : {}),
+        },
+        driverStartedAt,
+      ),
+    );
+  }
+  return [...activities.values()]
+    .map(({ calls, ...activity }) => {
+      for (const outcome of calls.values())
+        activity.toolCalls[outcome === "pending" ? "unresolved" : outcome]++;
+      return AgentActivitySchema.parse(activity);
+    })
+    .sort((a, b) => a.startedAt - b.startedAt || a.agent.id.localeCompare(b.agent.id));
+};

package/src/core/agent-events.ts

@@ -0,0 +1,17 @@
+import { z } from "zod";
+import { AgentAttributionSchema } from "./events.js";
+
+export const AGENT_SPAWN_KIND = "agent-spawn" as const;
+export const AGENT_COMPLETED_KIND = "agent-completed" as const;
+
+export const AgentSpawnPayloadSchema = z.object({
+  agent: AgentAttributionSchema.extend({ role: z.literal("subagent") }),
+  prompt: z.exactOptional(z.string()),
+});
+export type AgentSpawnPayload = z.infer<typeof AgentSpawnPayloadSchema>;
+
+export const AgentCompletedPayloadSchema = z.object({
+  agentId: z.string().min(1),
+  status: z.enum(["completed", "failed", "cancelled", "unknown"]),
+});
+export type AgentCompletedPayload = z.infer<typeof AgentCompletedPayloadSchema>;

package/src/core/driver-metrics.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const DriverMetricUnitSchema = z.enum([
+  "number",
+  "count",
+  "milliseconds",
+  "bytes",
+  "tokens",
+  "usd",
+  "ratio",
+  "percent",
+  "score",
+]);
+export type DriverMetricUnit = z.infer<typeof DriverMetricUnitSchema>;
+
+export const DriverMetricDescriptorSchema = z
+  .object({
+    unit: DriverMetricUnitSchema.default("number"),
+    label: z.string().min(1).max(120).optional(),
+    better: z.enum(["lower", "higher", "neutral"]).default("neutral"),
+  })
+  .strict();
+export type DriverMetricDescriptor = z.infer<typeof DriverMetricDescriptorSchema>;
+
+export const DriverMetricSchema = DriverMetricDescriptorSchema.extend({
+  value: z.number().finite(),
+}).strict();
+export type DriverMetric = z.infer<typeof DriverMetricSchema>;
+
+export const DriverMetricsSchema = z.record(z.string().min(1).max(120), DriverMetricSchema);
+export type DriverMetrics = z.infer<typeof DriverMetricsSchema>;
+
+const ReportedDriverMetricSchema = z.union([z.number().finite(), DriverMetricSchema]);
+export const ReportedDriverMetricsSchema = z.record(
+  z.string().min(1).max(120),
+  ReportedDriverMetricSchema,
+);
+export type ReportedDriverMetrics = z.input<typeof ReportedDriverMetricsSchema>;
+
+export const normalizeDriverMetrics = (value: unknown): DriverMetrics => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value).flatMap(([name, metric]) => {
+      if (name.length === 0 || name.length > 120) return [];
+      const parsed = DriverMetricSchema.safeParse(
+        typeof metric === "number" ? { value: metric } : metric,
+      ).data;
+      return parsed ? [[name, parsed]] : [];
+    }),
+  );
+};

package/src/core/events.ts

@@ -57,11 +57,22 @@ export const AnswerSchema = z.discriminatedUnion("kind", [
 ]);
 export type Answer = z.infer<typeof AnswerSchema>;
 
+export const AgentAttributionSchema = z.object({
+  id: z.string().min(1),
+  role: z.enum(["main", "subagent"]),
+  parentId: z.exactOptional(z.string().min(1)),
+  spawnToolUseId: z.exactOptional(z.string().min(1)),
+  name: z.exactOptional(z.string().min(1)),
+  model: z.exactOptional(z.string().min(1)),
+});
+export type AgentAttribution = z.infer<typeof AgentAttributionSchema>;
+
 export const ProgressSchema = z.object({
   id: z.string(),
   kind: z.string(),
   payload: z.unknown(),
   ts: z.number(),
+  agent: z.exactOptional(AgentAttributionSchema),
 });
 export type Progress = z.infer<typeof ProgressSchema>;
 
@@ -69,8 +80,8 @@ export const DoneSchema = z.object({
   exitReason: z.literal("done"),
   /**
    * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
-   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
-   * for domain-specific measurements.
+   * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
+   * preserving room for other domain-specific evidence.
    */
   summary: z.unknown().optional(),
 });

package/src/core/experiment.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
 import { ConfidenceConfigSchema } from "./statistics.js";
 import { TokenUsageSchema } from "./usage.js";
 import { formatZodIssues } from "./zod.js";
@@ -415,6 +416,7 @@ export const CandidateEvaluationSchema = z.object({
   passRate: z.number().min(0).max(1),
   casePassRates: z.record(z.string(), z.number().min(0).max(1)),
   metrics: z.record(z.string(), z.number()),
+  metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
   metricCalls: z.exactOptional(z.number().int().nonnegative()),
   judgeCalls: z.exactOptional(z.number().int().nonnegative()),
   unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),

package/src/core/index.ts

@@ -1,3 +1,6 @@
+export * from "./action-events.js";
+export * from "./agent-activity.js";
+export * from "./agent-events.js";
 export * from "./annotation.js";
 export * from "./answerer.js";
 export * from "./artifact-evidence.js";
@@ -6,6 +9,7 @@ export * from "./assertion.js";
 export * from "./case.js";
 export * from "./conversation-turn.js";
 export * from "./driver.js";
+export * from "./driver-metrics.js";
 export * from "./emitter.js";
 export * from "./events.js";
 export * from "./exec.js";

package/src/core/run.ts

@@ -1,6 +1,9 @@
 import { z } from "zod";
+import { ActionTimingSchema } from "./action-events.js";
+import { AgentActivitySchema } from "./agent-activity.js";
 import { EvalCaseSchema } from "./case.js";
 import { ArtifactSchema } from "./driver.js";
+import { DriverMetricsSchema } from "./driver-metrics.js";
 import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
 import { ToolOutcomeSummarySchema } from "./tool-events.js";
 import { RunUsageSchema } from "./usage.js";
@@ -112,6 +115,8 @@ export const RunMetadataSchema = z.object({
     }),
   ),
   cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
+  /** Driver-defined scalar measurements with display and optimization semantics. */
+  driverMetrics: z.exactOptional(DriverMetricsSchema),
   interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
   /** Legacy raw count of progress events whose kind is `tool-call`. */
   toolCallCount: z.exactOptional(z.number()),
@@ -124,6 +129,10 @@ export const RunMetadataSchema = z.object({
   ),
   /** Wall-clock per phase, derived from the lifecycle event stream. */
   phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
+  /** Driver-reported semantic action latency, measured from `start-driver`. */
+  actionTiming: z.exactOptional(ActionTimingSchema),
+  /** Per-agent lifecycle and activity derived from the attributed event stream. */
+  agentActivity: z.exactOptional(z.array(AgentActivitySchema)),
   /**
    * Harness/infrastructure failure that invalidates this run as candidate
    * evidence. A driver-reported subject failure intentionally leaves this

package/src/drivers/claude-code/parse-events.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import {
+  type AgentAttribution,
   type Interrupt,
   type ParsedDriverEvent,
   parseAgentUsage,
@@ -8,6 +9,39 @@ import {
   toolResultProgress,
 } from "../../core/index.js";
 
+const claudeAgent = (parentToolUseId?: string): AgentAttribution =>
+  parentToolUseId
+    ? {
+        id: `claude-code:${parentToolUseId}`,
+        role: "subagent",
+        parentId: "claude-code",
+        spawnToolUseId: parentToolUseId,
+      }
+    : { id: "claude-code", role: "main" };
+
+const isSubagentTool = (name: string): boolean => name === "Agent" || name === "Task";
+
+const spawnedClaudeAgent = (
+  toolUseId: string,
+  input: unknown,
+  parentToolUseId?: string,
+): AgentAttribution => {
+  const details =
+    typeof input === "object" && input !== null ? (input as Record<string, unknown>) : {};
+  const name = [details.name, details.subagent_type, details.agent, details.role].find(
+    (value): value is string => typeof value === "string" && value.length > 0,
+  );
+  const model = typeof details.model === "string" && details.model ? details.model : undefined;
+  return {
+    id: `claude-code:${toolUseId}`,
+    role: "subagent",
+    parentId: claudeAgent(parentToolUseId).id,
+    spawnToolUseId: toolUseId,
+    ...(name ? { name } : {}),
+    ...(model ? { model } : {}),
+  };
+};
+
 const ContentBlockSchema = z.discriminatedUnion("type", [
   z.object({ type: z.literal("text"), text: z.string() }),
   z.object({ type: z.literal("tool_use"), id: z.string(), name: z.string(), input: z.unknown() }),
@@ -88,6 +122,7 @@ type PendingTurn = {
   asked: boolean;
   thought: boolean;
   text?: string;
+  agent: AgentAttribution;
 };
 
 type AssistantContent = {
@@ -96,6 +131,7 @@ type AssistantContent = {
   asked: boolean;
   thought: boolean;
   firstText?: string;
+  agent: AgentAttribution;
 };
 
 const questionTextOf = (raw: unknown): string =>
@@ -225,6 +261,30 @@ class ClaudeEventParser {
       if (!block) continue;
       events.push(block);
       if (block.tag !== "progress") continue;
+      const resultPayload = block.progress.payload as {
+        name?: unknown;
+        toolUseId?: unknown;
+        isError?: unknown;
+      };
+      if (
+        typeof resultPayload.name === "string" &&
+        isSubagentTool(resultPayload.name) &&
+        typeof resultPayload.toolUseId === "string"
+      ) {
+        events.push({
+          tag: "progress",
+          progress: {
+            id: `agent-completed-${resultPayload.toolUseId}`,
+            kind: "agent-completed",
+            payload: {
+              agentId: `claude-code:${resultPayload.toolUseId}`,
+              status: resultPayload.isError === true ? "failed" : "completed",
+            },
+            agent: claudeAgent(),
+            ts: block.progress.ts,
+          },
+        });
+      }
       const text = (block.progress.payload as { text?: unknown }).text;
       const failure = typeof text === "string" ? toolInfrastructureFailure(text) : null;
       if (failure) {
@@ -244,20 +304,21 @@ class ClaudeEventParser {
 
     const toolUseId = block.tool_use_id;
     const content = block.content ?? [];
+    const progress = toolResultProgress(
+      this.eventSequence++,
+      {
+        name: this.toolNames.get(toolUseId) ?? "unknown",
+        toolUseId,
+        content,
+        isError: block.is_error === true,
+        text: toolResultText(content),
+        ...(parentToolUseId ? { parentToolUseId } : {}),
+      },
+      Date.now(),
+    );
     return {
       tag: "progress",
-      progress: toolResultProgress(
-        this.eventSequence++,
-        {
-          name: this.toolNames.get(toolUseId) ?? "unknown",
-          toolUseId,
-          content,
-          isError: block.is_error === true,
-          text: toolResultText(content),
-          ...(parentToolUseId ? { parentToolUseId } : {}),
-        },
-        Date.now(),
-      ),
+      progress: { ...progress, agent: claudeAgent(parentToolUseId) },
     };
   }
 
@@ -284,6 +345,7 @@ class ClaudeEventParser {
       tools: [],
       asked: false,
       thought: false,
+      agent: claudeAgent(event.parent_tool_use_id),
     };
     for (const rawBlock of event.message.content) {
       const block = ContentBlockSchema.safeParse(rawBlock).data;
@@ -323,6 +385,7 @@ class ClaudeEventParser {
         kind: "agent-message",
         payload: { text, ...(parentToolUseId ? { parentToolUseId } : {}) },
         ts: Date.now(),
+        agent: claudeAgent(parentToolUseId),
       },
     });
   }
@@ -354,8 +417,29 @@ class ClaudeEventParser {
           ...(parentToolUseId ? { parentToolUseId } : {}),
         },
         ts: Date.now(),
+        agent: claudeAgent(parentToolUseId),
       },
     });
+    if (isSubagentTool(block.name)) {
+      const agent = spawnedClaudeAgent(block.id, block.input, parentToolUseId);
+      const details =
+        typeof block.input === "object" && block.input !== null
+          ? (block.input as Record<string, unknown>)
+          : {};
+      content.events.push({
+        tag: "progress",
+        progress: {
+          id: `agent-spawn-${block.id}`,
+          kind: "agent-spawn",
+          payload: {
+            agent,
+            ...(typeof details.prompt === "string" ? { prompt: details.prompt } : {}),
+          },
+          agent: claudeAgent(parentToolUseId),
+          ts: Date.now(),
+        },
+      });
+    }
   }
 
   private recordTurn(
@@ -376,6 +460,7 @@ class ClaudeEventParser {
       tools: [],
       asked: false,
       thought: false,
+      agent: content.agent,
     };
     this.pendingTurn.usage = usage;
     this.pendingTurn.asked ||= content.asked;
@@ -451,6 +536,7 @@ class ClaudeEventParser {
         kind: "usage",
         payload: { ...this.pendingTurn.usage, ...(label ? { label } : {}) },
         ts: Date.now(),
+        agent: this.pendingTurn.agent,
       },
     };
     this.pendingTurn = null;