@vitest-evals/core 0.13.1 → 0.15.0

package/dist/index.js

@@ -29,7 +29,6 @@ __export(index_exports, {
   JsonPrimitiveSchema: () => JsonPrimitiveSchema,
   JsonValueSchema: () => JsonValueSchema,
   NormalizedErrorSchema: () => NormalizedErrorSchema,
-  NormalizedMessageSchema: () => NormalizedMessageSchema,
   NormalizedSessionSchema: () => NormalizedSessionSchema,
   NormalizedSpanAttributesSchema: () => NormalizedSpanAttributesSchema,
   NormalizedSpanEventSchema: () => NormalizedSpanEventSchema,
@@ -40,7 +39,11 @@ __export(index_exports, {
   ReportRunSchema: () => ReportRunSchema,
   ReportWorkspaceSchema: () => ReportWorkspaceSchema,
   TimingSummarySchema: () => TimingSummarySchema,
-  ToolCallRecordSchema: () => ToolCallRecordSchema,
+  ToolCallSchema: () => ToolCallSchema,
+  TranscriptEventSchema: () => TranscriptEventSchema,
+  TranscriptMessageEventSchema: () => TranscriptMessageEventSchema,
+  TranscriptToolCallEventSchema: () => TranscriptToolCallEventSchema,
+  TranscriptToolResultEventSchema: () => TranscriptToolResultEventSchema,
   UsageSummarySchema: () => UsageSummarySchema,
   VitestJsonAssertionSchema: () => VitestJsonAssertionSchema,
   VitestJsonFileSchema: () => VitestJsonFileSchema,
@@ -50,8 +53,12 @@ __export(index_exports, {
   assistantMessages: () => assistantMessages,
   collectReportWorkspace: () => collectReportWorkspace,
   failedSpans: () => failedSpans,
+  isMessageEvent: () => isMessageEvent,
+  isToolCallEvent: () => isToolCallEvent,
+  isToolResultEvent: () => isToolResultEvent,
   latestAssistantMessageContent: () => latestAssistantMessageContent,
   messagesByRole: () => messagesByRole,
+  messagesToTranscriptEvents: () => messagesToTranscriptEvents,
   parseReportWorkspace: () => parseReportWorkspace,
   parseVitestJsonReport: () => parseVitestJsonReport,
   readEvalTaskMeta: () => readEvalTaskMeta,
@@ -83,7 +90,7 @@ var JsonValueSchema = import_zod.z.lazy(
 var JsonObjectSchema = import_zod.z.record(import_zod.z.string(), JsonValueSchema);
 
 // src/harness/index.ts
-var import_zod3 = require("zod");
+var import_zod5 = require("zod");
 
 // src/schema-utils.ts
 var import_zod2 = require("zod");
@@ -114,10 +121,232 @@ function isRecord(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
 
+// src/harness/errors.ts
+var import_zod3 = require("zod");
+var NormalizedErrorSchema = import_zod3.z.object({
+  message: import_zod3.z.string(),
+  type: import_zod3.z.string().optional()
+}).catchall(JsonValueSchema);
+
+// src/harness/transcript.ts
+var import_zod4 = require("zod");
+var TranscriptToolCallEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("tool_call"),
+  id: import_zod4.z.string(),
+  name: import_zod4.z.string(),
+  arguments: JsonObjectSchema.optional(),
+  startedAt: import_zod4.z.string().optional(),
+  finishedAt: import_zod4.z.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptMessageEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("message"),
+  role: import_zod4.z.enum(["system", "user", "assistant"]),
+  content: JsonValueSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptToolResultEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("tool_result"),
+  toolCallId: import_zod4.z.string(),
+  name: import_zod4.z.string().optional(),
+  content: JsonValueSchema.optional(),
+  error: NormalizedErrorSchema.optional(),
+  startedAt: import_zod4.z.string().optional(),
+  finishedAt: import_zod4.z.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptEventSchema = import_zod4.z.discriminatedUnion("type", [
+  TranscriptMessageEventSchema,
+  TranscriptToolCallEventSchema,
+  TranscriptToolResultEventSchema
+]);
+function messagesToTranscriptEvents(messages) {
+  const events = [];
+  for (const [messageIndex, message] of messages.entries()) {
+    if (message.role === "tool") {
+      const partEvents2 = contentPartEvents(message);
+      if (partEvents2) {
+        events.push(...partEvents2);
+        continue;
+      }
+      if (!hasTopLevelToolCallId(message)) {
+        throw new TypeError("Tool result messages must include toolCallId.");
+      }
+      events.push(toolResultMessageEvent(message, message.toolCallId));
+      continue;
+    }
+    const partEvents = message.role === "assistant" ? contentPartEvents(message) : void 0;
+    const partEventsHaveToolCalls = partEvents?.some(
+      (event) => event.type === "tool_call"
+    );
+    if (partEvents) {
+      events.push(...partEvents);
+    } else if (message.content !== void 0) {
+      events.push({
+        type: "message",
+        role: message.role,
+        content: message.content,
+        ...message.metadata ? { metadata: message.metadata } : {}
+      });
+    }
+    if (message.role !== "assistant") {
+      continue;
+    }
+    if (partEventsHaveToolCalls) {
+      if ((message.toolCalls ?? []).length > 0) {
+        throw new TypeError(
+          "Assistant messages must not mix tool-call content parts with toolCalls."
+        );
+      }
+      continue;
+    }
+    const messageToolCalls = message.toolCalls ?? [];
+    for (const [toolIndex, toolCall] of messageToolCalls.entries()) {
+      const rawToolCall = toolCall;
+      if (Object.prototype.hasOwnProperty.call(rawToolCall, "result") || Object.prototype.hasOwnProperty.call(rawToolCall, "error")) {
+        throw new TypeError(
+          "Assistant tool calls must use separate tool result messages."
+        );
+      }
+      if (typeof toolCall.name !== "string" || toolCall.name.length === 0) {
+        throw new TypeError("Assistant tool calls must include name.");
+      }
+      const id = toolCall.id ?? `message-${messageIndex}:tool-call-${toolIndex}`;
+      events.push({
+        type: "tool_call",
+        id,
+        name: toolCall.name,
+        ...normalizeToolCallArguments(toolCall.arguments),
+        ...toolCall.startedAt ? { startedAt: toolCall.startedAt } : {},
+        ...toolCall.finishedAt ? { finishedAt: toolCall.finishedAt } : {},
+        ...toolCall.durationMs !== void 0 ? { durationMs: toolCall.durationMs } : {},
+        ...toolCall.metadata ? { metadata: toolCall.metadata } : {}
+      });
+    }
+  }
+  return events;
+}
+function contentPartEvents(message) {
+  if (!Array.isArray(message.content)) {
+    return void 0;
+  }
+  const events = [];
+  for (const part of message.content) {
+    if (!isJsonObject2(part) || typeof part.type !== "string") {
+      continue;
+    }
+    if (part.type === "text" && typeof part.text === "string") {
+      if (message.role === "tool") {
+        throw new TypeError("Text content parts require message role.");
+      }
+      events.push({
+        type: "message",
+        role: message.role,
+        content: part.text,
+        ...recordMetadata(message.metadata)
+      });
+      continue;
+    }
+    if (part.type === "tool-call") {
+      if (message.role !== "assistant" || typeof part.toolCallId !== "string" || typeof part.toolName !== "string") {
+        throw new TypeError(
+          "Tool-call content parts require assistant role, toolCallId, and toolName."
+        );
+      }
+      events.push({
+        type: "tool_call",
+        id: part.toolCallId,
+        name: part.toolName,
+        ...normalizeToolCallArguments(part.input),
+        ...jsonTimeFields(part),
+        ...jsonMetadata(part.metadata)
+      });
+      continue;
+    }
+    if (part.type === "tool-result") {
+      if (message.role !== "tool" || typeof part.toolCallId !== "string") {
+        throw new TypeError(
+          "Tool-result content parts require tool role and toolCallId."
+        );
+      }
+      if (hasTopLevelToolCallId(message)) {
+        throw new TypeError(
+          "Tool-result content parts must not include top-level toolCallId."
+        );
+      }
+      events.push({
+        type: "tool_result",
+        toolCallId: part.toolCallId,
+        ...typeof part.toolName === "string" ? { name: part.toolName } : {},
+        ...part.output !== void 0 ? { content: part.output } : {},
+        ...part.error ? { error: part.error } : {},
+        ...jsonTimeFields(part),
+        ...jsonMetadata(part.metadata)
+      });
+      continue;
+    }
+    throw new TypeError(
+      "Message content parts must use the harness message contract."
+    );
+  }
+  return events.length > 0 ? events : void 0;
+}
+function toolResultMessageEvent(message, toolCallId) {
+  return {
+    type: "tool_result",
+    toolCallId,
+    ...message.name ? { name: message.name } : {},
+    ...message.content !== void 0 ? { content: message.content } : {},
+    ...message.error ? { error: message.error } : {},
+    ...timeFields(message),
+    ...recordMetadata(message.metadata)
+  };
+}
+function hasTopLevelToolCallId(message) {
+  return typeof message.toolCallId === "string";
+}
+function timeFields(input) {
+  return {
+    ...input.startedAt ? { startedAt: input.startedAt } : {},
+    ...input.finishedAt ? { finishedAt: input.finishedAt } : {},
+    ...input.durationMs !== void 0 ? { durationMs: input.durationMs } : {}
+  };
+}
+function recordMetadata(metadata) {
+  return metadata ? { metadata } : {};
+}
+function jsonTimeFields(input) {
+  return {
+    ...typeof input.startedAt === "string" ? { startedAt: input.startedAt } : {},
+    ...typeof input.finishedAt === "string" ? { finishedAt: input.finishedAt } : {},
+    ...typeof input.durationMs === "number" ? { durationMs: input.durationMs } : {}
+  };
+}
+function jsonMetadata(value) {
+  return isJsonObject2(value) ? { metadata: value } : {};
+}
+function normalizeToolCallArguments(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? { arguments: value } : {};
+}
+function isJsonObject2(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function isMessageEvent(event) {
+  return event.type === "message";
+}
+function isToolCallEvent(event) {
+  return event.type === "tool_call";
+}
+function isToolResultEvent(event) {
+  return event.type === "tool_result";
+}
+
 // src/harness/index.ts
-var UsageSummarySchema = import_zod3.z.object({
-  provider: import_zod3.z.string().optional(),
-  model: import_zod3.z.string().optional(),
+var UsageSummarySchema = import_zod5.z.object({
+  provider: import_zod5.z.string().optional(),
+  model: import_zod5.z.string().optional(),
   inputTokens: FiniteNumberSchema.optional(),
   outputTokens: FiniteNumberSchema.optional(),
   reasoningTokens: FiniteNumberSchema.optional(),
@@ -126,81 +355,105 @@ var UsageSummarySchema = import_zod3.z.object({
   retries: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var TimingSummarySchema = import_zod3.z.object({
+var TimingSummarySchema = import_zod5.z.object({
   totalMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var NormalizedErrorSchema = import_zod3.z.object({
-  message: import_zod3.z.string(),
-  type: import_zod3.z.string().optional()
-}).catchall(JsonValueSchema);
-var ToolCallRecordSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  name: import_zod3.z.string(),
-  arguments: JsonObjectSchema.optional(),
-  result: JsonValueSchema.optional(),
-  error: NormalizedErrorSchema.optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
-  durationMs: FiniteNumberSchema.optional(),
-  metadata: JsonObjectSchema.optional()
-}).strict();
-var NormalizedMessageSchema = import_zod3.z.object({
-  role: import_zod3.z.enum(["system", "user", "assistant", "tool"]),
-  content: JsonValueSchema.optional(),
-  toolCalls: import_zod3.z.array(ToolCallRecordSchema).optional(),
-  metadata: JsonObjectSchema.optional()
+var ToolCallBaseSchema = import_zod5.z.object({
+  name: import_zod5.z.string(),
+  arguments: JsonObjectSchema.optional()
 }).strict();
-var NormalizedSessionSchema = import_zod3.z.object({
-  messages: import_zod3.z.array(NormalizedMessageSchema).default([]),
-  provider: import_zod3.z.string().optional(),
-  model: import_zod3.z.string().optional(),
+var ToolCallSchema = import_zod5.z.discriminatedUnion("status", [
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("pending")
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("ok"),
+    result: JsonValueSchema.optional()
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("error"),
+    error: NormalizedErrorSchema
+  }).strict()
+]);
+var NormalizedSessionSchema = import_zod5.z.object({
+  events: import_zod5.z.array(TranscriptEventSchema),
+  provider: import_zod5.z.string().optional(),
+  model: import_zod5.z.string().optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
 var NormalizedSpanAttributesSchema = JsonObjectSchema;
-var NormalizedSpanEventSchema = import_zod3.z.object({
-  name: import_zod3.z.string(),
-  timestamp: import_zod3.z.string().optional(),
+var NormalizedSpanEventSchema = import_zod5.z.object({
+  name: import_zod5.z.string(),
+  timestamp: import_zod5.z.string().optional(),
   attributes: NormalizedSpanAttributesSchema.optional()
 }).strict();
-var NormalizedSpanSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  traceId: import_zod3.z.string().optional(),
-  parentId: import_zod3.z.string().optional(),
-  name: import_zod3.z.string(),
-  kind: import_zod3.z.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
+var NormalizedSpanSchema = import_zod5.z.object({
+  id: import_zod5.z.string().optional(),
+  traceId: import_zod5.z.string().optional(),
+  parentId: import_zod5.z.string().optional(),
+  name: import_zod5.z.string(),
+  kind: import_zod5.z.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
+  startedAt: import_zod5.z.string().optional(),
+  finishedAt: import_zod5.z.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
-  status: import_zod3.z.enum(["ok", "error"]).optional(),
+  status: import_zod5.z.enum(["ok", "error"]).optional(),
   error: NormalizedErrorSchema.optional(),
   attributes: NormalizedSpanAttributesSchema.optional(),
-  events: import_zod3.z.array(NormalizedSpanEventSchema).optional()
+  events: import_zod5.z.array(NormalizedSpanEventSchema).optional()
 }).strict();
-var NormalizedTraceSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  name: import_zod3.z.string().optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
+var NormalizedTraceSchema = import_zod5.z.object({
+  id: import_zod5.z.string().optional(),
+  name: import_zod5.z.string().optional(),
+  startedAt: import_zod5.z.string().optional(),
+  finishedAt: import_zod5.z.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional(),
-  spans: import_zod3.z.array(NormalizedSpanSchema).default([])
+  spans: import_zod5.z.array(NormalizedSpanSchema)
 }).strict();
-var HarnessRunSchema = import_zod3.z.object({
+var HarnessRunSchema = import_zod5.z.object({
   output: JsonValueSchema.optional(),
   session: NormalizedSessionSchema,
   usage: UsageSummarySchema,
   timings: TimingSummarySchema.optional(),
   artifacts: JsonObjectSchema.optional(),
-  traces: import_zod3.z.array(NormalizedTraceSchema).optional(),
-  errors: import_zod3.z.array(JsonObjectSchema).default([])
+  traces: import_zod5.z.array(NormalizedTraceSchema).optional(),
+  errors: import_zod5.z.array(JsonObjectSchema)
 }).strict();
 
 // src/harness/helpers.ts
 function toolCalls(source) {
-  return sessionFrom(source).messages.flatMap(
-    (message) => message.toolCalls ?? []
-  );
+  const resultsById = /* @__PURE__ */ new Map();
+  for (const message of toolResultsFromSource(source)) {
+    if (!resultsById.has(message.toolCallId)) {
+      resultsById.set(message.toolCallId, message);
+    }
+  }
+  return toolCallsFromSource(source).map((call) => {
+    const result = resultsById.get(call.id);
+    const normalizedCall = {
+      name: call.name,
+      ...call.arguments ? { arguments: call.arguments } : {}
+    };
+    if (!result) {
+      return {
+        ...normalizedCall,
+        status: "pending"
+      };
+    }
+    if (result.error) {
+      return {
+        ...normalizedCall,
+        status: "error",
+        error: result.error
+      };
+    }
+    return {
+      ...normalizedCall,
+      status: "ok",
+      ...result.content !== void 0 ? { result: result.content } : {}
+    };
+  });
 }
 function spans(source) {
   return spansFrom(source);
@@ -217,8 +470,8 @@ function failedSpans(source) {
   );
 }
 function messagesByRole(source, role) {
-  return sessionFrom(source).messages.filter(
-    (message) => message.role === role
+  return sessionFrom(source).events.filter(
+    (event) => event.type === "message" && event.role === role
   );
 }
 function systemMessages(source) {
@@ -234,11 +487,17 @@ function latestAssistantMessageContent(source) {
   return [...messagesByRoleFromSource(source, "assistant")].reverse().find(hasNonEmptyMessageContent)?.content;
 }
 function toolMessages(source) {
-  return messagesByRoleFromSource(source, "tool");
+  return toolResultsFromSource(source);
 }
 function sessionFrom(source) {
   return "session" in source ? source.session : source;
 }
+function toolResultsFromSource(source) {
+  return sessionFrom(source).events.filter(isToolResultEvent);
+}
+function toolCallsFromSource(source) {
+  return sessionFrom(source).events.filter(isToolCallEvent);
+}
 function tracesFrom(source) {
   if (source === void 0) {
     return [];
@@ -255,8 +514,8 @@ function isTraceList(source) {
   return Array.isArray(source);
 }
 function messagesByRoleFromSource(source, role) {
-  return sessionFrom(source).messages.filter(
-    (message) => message.role === role
+  return sessionFrom(source).events.filter(
+    (event) => event.type === "message" && event.role === role
   );
 }
 function hasNonEmptyMessageContent(message) {
@@ -264,71 +523,45 @@ function hasNonEmptyMessageContent(message) {
 }
 
 // src/report/metadata.ts
-var import_zod4 = require("zod");
-var HarnessMetaSchema = import_zod4.z.object({
-  name: import_zod4.z.string().optional(),
+var import_zod6 = require("zod");
+var HarnessMetaSchema = import_zod6.z.object({
+  name: import_zod6.z.string().optional(),
   run: HarnessRunSchema.optional()
 }).strict();
-var EvalScoreSchema = import_zod4.z.object({
-  name: import_zod4.z.string().optional(),
+var EvalScoreSchema = import_zod6.z.object({
+  name: import_zod6.z.string().optional(),
   score: NullableFiniteNumberSchema,
   metadata: JsonObjectSchema.optional()
 }).strict();
-var EvalMetaSchema = import_zod4.z.object({
-  scores: import_zod4.z.array(EvalScoreSchema).optional(),
+var EvalMetaSchema = import_zod6.z.object({
+  scores: import_zod6.z.array(EvalScoreSchema).optional(),
   avgScore: NullableFiniteNumberSchema,
   output: JsonValueSchema.optional(),
-  thresholdFailed: import_zod4.z.boolean().optional(),
-  toolCalls: import_zod4.z.array(ToolCallRecordSchema).optional()
+  thresholdFailed: import_zod6.z.boolean().optional(),
+  toolCalls: import_zod6.z.array(ToolCallSchema).optional()
 }).strict();
-var EvalTaskMetaSchema = import_zod4.z.object({
+var EvalTaskMetaSchema = import_zod6.z.object({
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var LenientToolCallRecordSchema = ToolCallRecordSchema.strip();
-var LenientMessageSchema = NormalizedMessageSchema.extend({
-  toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
-var LenientSessionSchema = NormalizedSessionSchema.extend({
-  messages: import_zod4.z.array(LenientMessageSchema).default([]).catch([])
-}).strip();
-var LenientSpanEventSchema = NormalizedSpanEventSchema.strip();
-var LenientSpanSchema = NormalizedSpanSchema.extend({
-  events: import_zod4.z.array(LenientSpanEventSchema).optional().catch(void 0)
-}).strip();
-var LenientTraceSchema = NormalizedTraceSchema.extend({
-  spans: import_zod4.z.array(LenientSpanSchema).default([]).catch([])
-}).strip();
-var LenientHarnessRunSchema = HarnessRunSchema.extend({
-  session: LenientSessionSchema,
-  usage: UsageSummarySchema.strip().default({}),
-  timings: TimingSummarySchema.strip().optional(),
-  traces: import_zod4.z.array(LenientTraceSchema).optional().catch(void 0)
-}).strip();
-var LenientHarnessMetaSchema = HarnessMetaSchema.extend({
-  run: LenientHarnessRunSchema.optional().catch(void 0)
-}).strip();
-var LenientEvalScoreSchema = EvalScoreSchema.strip();
-var LenientEvalMetaSchema = EvalMetaSchema.extend({
-  scores: import_zod4.z.array(LenientEvalScoreSchema).optional().catch(void 0),
-  toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
 function readEvalTaskMeta(input) {
   if (!isJsonObject(input)) {
     return void 0;
   }
-  const evalResult = LenientEvalMetaSchema.safeParse(input.eval);
-  const harnessResult = LenientHarnessMetaSchema.safeParse(input.harness);
   const meta = {
-    ...evalResult.success && input.eval !== void 0 ? { eval: evalResult.data } : {},
-    ...harnessResult.success && input.harness !== void 0 ? { harness: harnessResult.data } : {}
+    ...input.eval !== void 0 ? { eval: input.eval } : {},
+    ...input.harness !== void 0 ? { harness: input.harness } : {}
   };
-  return meta.eval || meta.harness ? meta : void 0;
+  if (!("eval" in meta) && !("harness" in meta)) {
+    return void 0;
+  }
+  const result = EvalTaskMetaSchema.safeParse(meta);
+  return result.success ? result.data : void 0;
 }
 
 // src/report/vitest-json.ts
-var import_zod5 = require("zod");
-var VitestJsonStatusSchema = import_zod5.z.enum([
+var import_zod7 = require("zod");
+var VitestJsonStatusSchema = import_zod7.z.enum([
   "passed",
   "failed",
   "skipped",
@@ -336,53 +569,53 @@ var VitestJsonStatusSchema = import_zod5.z.enum([
   "todo",
   "disabled"
 ]);
-var VitestJsonLocationSchema = import_zod5.z.object({
+var VitestJsonLocationSchema = import_zod7.z.object({
   line: FiniteNumberSchema,
   column: FiniteNumberSchema
 }).passthrough();
-var VitestJsonAssertionSchema = import_zod5.z.object({
-  ancestorTitles: import_zod5.z.array(import_zod5.z.string()).default([]),
-  fullName: import_zod5.z.string(),
+var VitestJsonAssertionSchema = import_zod7.z.object({
+  ancestorTitles: import_zod7.z.array(import_zod7.z.string()).default([]),
+  fullName: import_zod7.z.string(),
   status: VitestJsonStatusSchema,
-  title: import_zod5.z.string(),
-  meta: import_zod5.z.unknown().optional(),
+  title: import_zod7.z.string(),
+  meta: import_zod7.z.unknown().optional(),
   duration: FiniteNumberSchema.nullable().optional(),
-  failureMessages: import_zod5.z.array(import_zod5.z.string()).nullable().optional(),
+  failureMessages: import_zod7.z.array(import_zod7.z.string()).nullable().optional(),
   location: VitestJsonLocationSchema.nullable().optional(),
-  tags: import_zod5.z.array(import_zod5.z.string()).optional()
+  tags: import_zod7.z.array(import_zod7.z.string()).optional()
 }).passthrough();
-var VitestJsonFileSchema = import_zod5.z.object({
-  message: import_zod5.z.string(),
-  name: import_zod5.z.string(),
-  status: import_zod5.z.enum(["failed", "passed"]),
+var VitestJsonFileSchema = import_zod7.z.object({
+  message: import_zod7.z.string(),
+  name: import_zod7.z.string(),
+  status: import_zod7.z.enum(["failed", "passed"]),
   startTime: OptionalFiniteNumberSchema,
   endTime: OptionalFiniteNumberSchema,
-  assertionResults: import_zod5.z.array(VitestJsonAssertionSchema).default([])
+  assertionResults: import_zod7.z.array(VitestJsonAssertionSchema).default([])
 }).passthrough();
-var VitestJsonReportSchema = import_zod5.z.object({
+var VitestJsonReportSchema = import_zod7.z.object({
   numFailedTests: FiniteNumberSchema,
   numPassedTests: FiniteNumberSchema,
   numPendingTests: FiniteNumberSchema,
   numTodoTests: FiniteNumberSchema,
   numTotalTests: FiniteNumberSchema,
   startTime: FiniteNumberSchema,
-  success: import_zod5.z.boolean(),
-  testResults: import_zod5.z.array(VitestJsonFileSchema).default([])
+  success: import_zod7.z.boolean(),
+  testResults: import_zod7.z.array(VitestJsonFileSchema).default([])
 }).passthrough();
 function parseVitestJsonReport(input) {
   return parseWithSchema(VitestJsonReportSchema, input, "Vitest JSON report");
 }
 
 // src/report/workspace.ts
-var import_zod6 = require("zod");
+var import_zod8 = require("zod");
 var REPORT_WORKSPACE_SCHEMA_VERSION = 1;
-var ReportRunSchema = import_zod6.z.object({
-  id: import_zod6.z.string(),
-  source: import_zod6.z.string().optional(),
-  status: import_zod6.z.enum(["passed", "failed"]),
+var ReportRunSchema = import_zod8.z.object({
+  id: import_zod8.z.string(),
+  source: import_zod8.z.string().optional(),
+  status: import_zod8.z.enum(["passed", "failed"]),
   startedAt: FiniteNumberSchema.optional(),
   durationMs: FiniteNumberSchema.optional(),
-  totals: import_zod6.z.object({
+  totals: import_zod8.z.object({
     total: FiniteNumberSchema,
     passed: FiniteNumberSchema,
     failed: FiniteNumberSchema,
@@ -392,28 +625,28 @@ var ReportRunSchema = import_zod6.z.object({
     evalFailed: FiniteNumberSchema
   })
 }).strict();
-var ReportCaseSchema = import_zod6.z.object({
-  id: import_zod6.z.string(),
-  runId: import_zod6.z.string(),
-  source: import_zod6.z.string().optional(),
-  file: import_zod6.z.string(),
-  displayFile: import_zod6.z.string(),
-  title: import_zod6.z.string(),
-  fullName: import_zod6.z.string(),
-  ancestorTitles: import_zod6.z.array(import_zod6.z.string()),
-  tags: import_zod6.z.array(import_zod6.z.string()).optional(),
-  displayName: import_zod6.z.string(),
+var ReportCaseSchema = import_zod8.z.object({
+  id: import_zod8.z.string(),
+  runId: import_zod8.z.string(),
+  source: import_zod8.z.string().optional(),
+  file: import_zod8.z.string(),
+  displayFile: import_zod8.z.string(),
+  title: import_zod8.z.string(),
+  fullName: import_zod8.z.string(),
+  ancestorTitles: import_zod8.z.array(import_zod8.z.string()),
+  tags: import_zod8.z.array(import_zod8.z.string()).optional(),
+  displayName: import_zod8.z.string(),
   status: VitestJsonStatusSchema,
   durationMs: FiniteNumberSchema.optional(),
   location: VitestJsonLocationSchema.optional(),
-  failureMessages: import_zod6.z.array(import_zod6.z.string()).default([]),
+  failureMessages: import_zod8.z.array(import_zod8.z.string()).default([]),
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var ReportWorkspaceSchema = import_zod6.z.object({
-  schemaVersion: import_zod6.z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
-  runs: import_zod6.z.array(ReportRunSchema),
-  cases: import_zod6.z.array(ReportCaseSchema)
+var ReportWorkspaceSchema = import_zod8.z.object({
+  schemaVersion: import_zod8.z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
+  runs: import_zod8.z.array(ReportRunSchema),
+  cases: import_zod8.z.array(ReportCaseSchema)
 }).strict();
 function parseReportWorkspace(input) {
   return parseWithSchema(ReportWorkspaceSchema, input, "report workspace");
@@ -432,6 +665,11 @@ function collectReportWorkspace(input, options = {}) {
         if (!meta) {
           continue;
         }
+        const evalMeta = meta.eval ?? (meta.harness ? {
+          avgScore: assertion.status === "passed" ? 1 : assertion.status === "failed" ? 0 : null,
+          scores: [],
+          thresholdFailed: false
+        } : void 0);
         runCases.push({
           id: createCaseId(runId, file.name, assertion),
           runId,
@@ -447,7 +685,7 @@ function collectReportWorkspace(input, options = {}) {
           ...typeof assertion.duration === "number" ? { durationMs: assertion.duration } : {},
           ...assertion.location ? { location: assertion.location } : {},
           failureMessages: assertion.failureMessages ?? [],
-          ...meta.eval ? { eval: meta.eval } : {},
+          ...evalMeta ? { eval: evalMeta } : {},
           ...meta.harness ? { harness: meta.harness } : {}
         });
       }
@@ -547,7 +785,6 @@ function normalizeReportPath(path, workspace) {
   JsonPrimitiveSchema,
   JsonValueSchema,
   NormalizedErrorSchema,
-  NormalizedMessageSchema,
   NormalizedSessionSchema,
   NormalizedSpanAttributesSchema,
   NormalizedSpanEventSchema,
@@ -558,7 +795,11 @@ function normalizeReportPath(path, workspace) {
   ReportRunSchema,
   ReportWorkspaceSchema,
   TimingSummarySchema,
-  ToolCallRecordSchema,
+  ToolCallSchema,
+  TranscriptEventSchema,
+  TranscriptMessageEventSchema,
+  TranscriptToolCallEventSchema,
+  TranscriptToolResultEventSchema,
   UsageSummarySchema,
   VitestJsonAssertionSchema,
   VitestJsonFileSchema,
@@ -568,8 +809,12 @@ function normalizeReportPath(path, workspace) {
   assistantMessages,
   collectReportWorkspace,
   failedSpans,
+  isMessageEvent,
+  isToolCallEvent,
+  isToolResultEvent,
   latestAssistantMessageContent,
   messagesByRole,
+  messagesToTranscriptEvents,
   parseReportWorkspace,
   parseVitestJsonReport,
   readEvalTaskMeta,