@vitest-evals/core 0.13.1 → 0.15.0

package/dist/node.js

@@ -32,13 +32,13 @@ var import_promises = require("fs/promises");
 var import_node_path = require("path");
 
 // src/report/workspace.ts
-var import_zod6 = require("zod");
+var import_zod8 = require("zod");
 
 // src/report/metadata.ts
-var import_zod4 = require("zod");
+var import_zod6 = require("zod");
 
 // src/harness/index.ts
-var import_zod3 = require("zod");
+var import_zod5 = require("zod");
 
 // src/json.ts
 var import_zod = require("zod");
@@ -86,10 +86,52 @@ function isRecord(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
 
+// src/harness/errors.ts
+var import_zod3 = require("zod");
+var NormalizedErrorSchema = import_zod3.z.object({
+  message: import_zod3.z.string(),
+  type: import_zod3.z.string().optional()
+}).catchall(JsonValueSchema);
+
+// src/harness/transcript.ts
+var import_zod4 = require("zod");
+var TranscriptToolCallEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("tool_call"),
+  id: import_zod4.z.string(),
+  name: import_zod4.z.string(),
+  arguments: JsonObjectSchema.optional(),
+  startedAt: import_zod4.z.string().optional(),
+  finishedAt: import_zod4.z.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptMessageEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("message"),
+  role: import_zod4.z.enum(["system", "user", "assistant"]),
+  content: JsonValueSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptToolResultEventSchema = import_zod4.z.object({
+  type: import_zod4.z.literal("tool_result"),
+  toolCallId: import_zod4.z.string(),
+  name: import_zod4.z.string().optional(),
+  content: JsonValueSchema.optional(),
+  error: NormalizedErrorSchema.optional(),
+  startedAt: import_zod4.z.string().optional(),
+  finishedAt: import_zod4.z.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptEventSchema = import_zod4.z.discriminatedUnion("type", [
+  TranscriptMessageEventSchema,
+  TranscriptToolCallEventSchema,
+  TranscriptToolResultEventSchema
+]);
+
 // src/harness/index.ts
-var UsageSummarySchema = import_zod3.z.object({
-  provider: import_zod3.z.string().optional(),
-  model: import_zod3.z.string().optional(),
+var UsageSummarySchema = import_zod5.z.object({
+  provider: import_zod5.z.string().optional(),
+  model: import_zod5.z.string().optional(),
   inputTokens: FiniteNumberSchema.optional(),
   outputTokens: FiniteNumberSchema.optional(),
   reasoningTokens: FiniteNumberSchema.optional(),
@@ -98,141 +140,111 @@ var UsageSummarySchema = import_zod3.z.object({
   retries: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var TimingSummarySchema = import_zod3.z.object({
+var TimingSummarySchema = import_zod5.z.object({
   totalMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var NormalizedErrorSchema = import_zod3.z.object({
-  message: import_zod3.z.string(),
-  type: import_zod3.z.string().optional()
-}).catchall(JsonValueSchema);
-var ToolCallRecordSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  name: import_zod3.z.string(),
-  arguments: JsonObjectSchema.optional(),
-  result: JsonValueSchema.optional(),
-  error: NormalizedErrorSchema.optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
-  durationMs: FiniteNumberSchema.optional(),
-  metadata: JsonObjectSchema.optional()
-}).strict();
-var NormalizedMessageSchema = import_zod3.z.object({
-  role: import_zod3.z.enum(["system", "user", "assistant", "tool"]),
-  content: JsonValueSchema.optional(),
-  toolCalls: import_zod3.z.array(ToolCallRecordSchema).optional(),
-  metadata: JsonObjectSchema.optional()
+var ToolCallBaseSchema = import_zod5.z.object({
+  name: import_zod5.z.string(),
+  arguments: JsonObjectSchema.optional()
 }).strict();
-var NormalizedSessionSchema = import_zod3.z.object({
-  messages: import_zod3.z.array(NormalizedMessageSchema).default([]),
-  provider: import_zod3.z.string().optional(),
-  model: import_zod3.z.string().optional(),
+var ToolCallSchema = import_zod5.z.discriminatedUnion("status", [
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("pending")
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("ok"),
+    result: JsonValueSchema.optional()
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: import_zod5.z.literal("error"),
+    error: NormalizedErrorSchema
+  }).strict()
+]);
+var NormalizedSessionSchema = import_zod5.z.object({
+  events: import_zod5.z.array(TranscriptEventSchema),
+  provider: import_zod5.z.string().optional(),
+  model: import_zod5.z.string().optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
 var NormalizedSpanAttributesSchema = JsonObjectSchema;
-var NormalizedSpanEventSchema = import_zod3.z.object({
-  name: import_zod3.z.string(),
-  timestamp: import_zod3.z.string().optional(),
+var NormalizedSpanEventSchema = import_zod5.z.object({
+  name: import_zod5.z.string(),
+  timestamp: import_zod5.z.string().optional(),
   attributes: NormalizedSpanAttributesSchema.optional()
 }).strict();
-var NormalizedSpanSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  traceId: import_zod3.z.string().optional(),
-  parentId: import_zod3.z.string().optional(),
-  name: import_zod3.z.string(),
-  kind: import_zod3.z.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
+var NormalizedSpanSchema = import_zod5.z.object({
+  id: import_zod5.z.string().optional(),
+  traceId: import_zod5.z.string().optional(),
+  parentId: import_zod5.z.string().optional(),
+  name: import_zod5.z.string(),
+  kind: import_zod5.z.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
+  startedAt: import_zod5.z.string().optional(),
+  finishedAt: import_zod5.z.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
-  status: import_zod3.z.enum(["ok", "error"]).optional(),
+  status: import_zod5.z.enum(["ok", "error"]).optional(),
   error: NormalizedErrorSchema.optional(),
   attributes: NormalizedSpanAttributesSchema.optional(),
-  events: import_zod3.z.array(NormalizedSpanEventSchema).optional()
+  events: import_zod5.z.array(NormalizedSpanEventSchema).optional()
 }).strict();
-var NormalizedTraceSchema = import_zod3.z.object({
-  id: import_zod3.z.string().optional(),
-  name: import_zod3.z.string().optional(),
-  startedAt: import_zod3.z.string().optional(),
-  finishedAt: import_zod3.z.string().optional(),
+var NormalizedTraceSchema = import_zod5.z.object({
+  id: import_zod5.z.string().optional(),
+  name: import_zod5.z.string().optional(),
+  startedAt: import_zod5.z.string().optional(),
+  finishedAt: import_zod5.z.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional(),
-  spans: import_zod3.z.array(NormalizedSpanSchema).default([])
+  spans: import_zod5.z.array(NormalizedSpanSchema)
 }).strict();
-var HarnessRunSchema = import_zod3.z.object({
+var HarnessRunSchema = import_zod5.z.object({
   output: JsonValueSchema.optional(),
   session: NormalizedSessionSchema,
   usage: UsageSummarySchema,
   timings: TimingSummarySchema.optional(),
   artifacts: JsonObjectSchema.optional(),
-  traces: import_zod3.z.array(NormalizedTraceSchema).optional(),
-  errors: import_zod3.z.array(JsonObjectSchema).default([])
+  traces: import_zod5.z.array(NormalizedTraceSchema).optional(),
+  errors: import_zod5.z.array(JsonObjectSchema)
 }).strict();
 
 // src/report/metadata.ts
-var HarnessMetaSchema = import_zod4.z.object({
-  name: import_zod4.z.string().optional(),
+var HarnessMetaSchema = import_zod6.z.object({
+  name: import_zod6.z.string().optional(),
   run: HarnessRunSchema.optional()
 }).strict();
-var EvalScoreSchema = import_zod4.z.object({
-  name: import_zod4.z.string().optional(),
+var EvalScoreSchema = import_zod6.z.object({
+  name: import_zod6.z.string().optional(),
   score: NullableFiniteNumberSchema,
   metadata: JsonObjectSchema.optional()
 }).strict();
-var EvalMetaSchema = import_zod4.z.object({
-  scores: import_zod4.z.array(EvalScoreSchema).optional(),
+var EvalMetaSchema = import_zod6.z.object({
+  scores: import_zod6.z.array(EvalScoreSchema).optional(),
   avgScore: NullableFiniteNumberSchema,
   output: JsonValueSchema.optional(),
-  thresholdFailed: import_zod4.z.boolean().optional(),
-  toolCalls: import_zod4.z.array(ToolCallRecordSchema).optional()
+  thresholdFailed: import_zod6.z.boolean().optional(),
+  toolCalls: import_zod6.z.array(ToolCallSchema).optional()
 }).strict();
-var EvalTaskMetaSchema = import_zod4.z.object({
+var EvalTaskMetaSchema = import_zod6.z.object({
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var LenientToolCallRecordSchema = ToolCallRecordSchema.strip();
-var LenientMessageSchema = NormalizedMessageSchema.extend({
-  toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
-var LenientSessionSchema = NormalizedSessionSchema.extend({
-  messages: import_zod4.z.array(LenientMessageSchema).default([]).catch([])
-}).strip();
-var LenientSpanEventSchema = NormalizedSpanEventSchema.strip();
-var LenientSpanSchema = NormalizedSpanSchema.extend({
-  events: import_zod4.z.array(LenientSpanEventSchema).optional().catch(void 0)
-}).strip();
-var LenientTraceSchema = NormalizedTraceSchema.extend({
-  spans: import_zod4.z.array(LenientSpanSchema).default([]).catch([])
-}).strip();
-var LenientHarnessRunSchema = HarnessRunSchema.extend({
-  session: LenientSessionSchema,
-  usage: UsageSummarySchema.strip().default({}),
-  timings: TimingSummarySchema.strip().optional(),
-  traces: import_zod4.z.array(LenientTraceSchema).optional().catch(void 0)
-}).strip();
-var LenientHarnessMetaSchema = HarnessMetaSchema.extend({
-  run: LenientHarnessRunSchema.optional().catch(void 0)
-}).strip();
-var LenientEvalScoreSchema = EvalScoreSchema.strip();
-var LenientEvalMetaSchema = EvalMetaSchema.extend({
-  scores: import_zod4.z.array(LenientEvalScoreSchema).optional().catch(void 0),
-  toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
 function readEvalTaskMeta(input) {
   if (!isJsonObject(input)) {
     return void 0;
   }
-  const evalResult = LenientEvalMetaSchema.safeParse(input.eval);
-  const harnessResult = LenientHarnessMetaSchema.safeParse(input.harness);
   const meta = {
-    ...evalResult.success && input.eval !== void 0 ? { eval: evalResult.data } : {},
-    ...harnessResult.success && input.harness !== void 0 ? { harness: harnessResult.data } : {}
+    ...input.eval !== void 0 ? { eval: input.eval } : {},
+    ...input.harness !== void 0 ? { harness: input.harness } : {}
   };
-  return meta.eval || meta.harness ? meta : void 0;
+  if (!("eval" in meta) && !("harness" in meta)) {
+    return void 0;
+  }
+  const result = EvalTaskMetaSchema.safeParse(meta);
+  return result.success ? result.data : void 0;
 }
 
 // src/report/vitest-json.ts
-var import_zod5 = require("zod");
-var VitestJsonStatusSchema = import_zod5.z.enum([
+var import_zod7 = require("zod");
+var VitestJsonStatusSchema = import_zod7.z.enum([
   "passed",
   "failed",
   "skipped",
@@ -240,38 +252,38 @@ var VitestJsonStatusSchema = import_zod5.z.enum([
   "todo",
   "disabled"
 ]);
-var VitestJsonLocationSchema = import_zod5.z.object({
+var VitestJsonLocationSchema = import_zod7.z.object({
   line: FiniteNumberSchema,
   column: FiniteNumberSchema
 }).passthrough();
-var VitestJsonAssertionSchema = import_zod5.z.object({
-  ancestorTitles: import_zod5.z.array(import_zod5.z.string()).default([]),
-  fullName: import_zod5.z.string(),
+var VitestJsonAssertionSchema = import_zod7.z.object({
+  ancestorTitles: import_zod7.z.array(import_zod7.z.string()).default([]),
+  fullName: import_zod7.z.string(),
   status: VitestJsonStatusSchema,
-  title: import_zod5.z.string(),
-  meta: import_zod5.z.unknown().optional(),
+  title: import_zod7.z.string(),
+  meta: import_zod7.z.unknown().optional(),
   duration: FiniteNumberSchema.nullable().optional(),
-  failureMessages: import_zod5.z.array(import_zod5.z.string()).nullable().optional(),
+  failureMessages: import_zod7.z.array(import_zod7.z.string()).nullable().optional(),
   location: VitestJsonLocationSchema.nullable().optional(),
-  tags: import_zod5.z.array(import_zod5.z.string()).optional()
+  tags: import_zod7.z.array(import_zod7.z.string()).optional()
 }).passthrough();
-var VitestJsonFileSchema = import_zod5.z.object({
-  message: import_zod5.z.string(),
-  name: import_zod5.z.string(),
-  status: import_zod5.z.enum(["failed", "passed"]),
+var VitestJsonFileSchema = import_zod7.z.object({
+  message: import_zod7.z.string(),
+  name: import_zod7.z.string(),
+  status: import_zod7.z.enum(["failed", "passed"]),
   startTime: OptionalFiniteNumberSchema,
   endTime: OptionalFiniteNumberSchema,
-  assertionResults: import_zod5.z.array(VitestJsonAssertionSchema).default([])
+  assertionResults: import_zod7.z.array(VitestJsonAssertionSchema).default([])
 }).passthrough();
-var VitestJsonReportSchema = import_zod5.z.object({
+var VitestJsonReportSchema = import_zod7.z.object({
   numFailedTests: FiniteNumberSchema,
   numPassedTests: FiniteNumberSchema,
   numPendingTests: FiniteNumberSchema,
   numTodoTests: FiniteNumberSchema,
   numTotalTests: FiniteNumberSchema,
   startTime: FiniteNumberSchema,
-  success: import_zod5.z.boolean(),
-  testResults: import_zod5.z.array(VitestJsonFileSchema).default([])
+  success: import_zod7.z.boolean(),
+  testResults: import_zod7.z.array(VitestJsonFileSchema).default([])
 }).passthrough();
 function parseVitestJsonReport(input) {
   return parseWithSchema(VitestJsonReportSchema, input, "Vitest JSON report");
@@ -279,13 +291,13 @@ function parseVitestJsonReport(input) {
 
 // src/report/workspace.ts
 var REPORT_WORKSPACE_SCHEMA_VERSION = 1;
-var ReportRunSchema = import_zod6.z.object({
-  id: import_zod6.z.string(),
-  source: import_zod6.z.string().optional(),
-  status: import_zod6.z.enum(["passed", "failed"]),
+var ReportRunSchema = import_zod8.z.object({
+  id: import_zod8.z.string(),
+  source: import_zod8.z.string().optional(),
+  status: import_zod8.z.enum(["passed", "failed"]),
   startedAt: FiniteNumberSchema.optional(),
   durationMs: FiniteNumberSchema.optional(),
-  totals: import_zod6.z.object({
+  totals: import_zod8.z.object({
     total: FiniteNumberSchema,
     passed: FiniteNumberSchema,
     failed: FiniteNumberSchema,
@@ -295,28 +307,28 @@ var ReportRunSchema = import_zod6.z.object({
     evalFailed: FiniteNumberSchema
   })
 }).strict();
-var ReportCaseSchema = import_zod6.z.object({
-  id: import_zod6.z.string(),
-  runId: import_zod6.z.string(),
-  source: import_zod6.z.string().optional(),
-  file: import_zod6.z.string(),
-  displayFile: import_zod6.z.string(),
-  title: import_zod6.z.string(),
-  fullName: import_zod6.z.string(),
-  ancestorTitles: import_zod6.z.array(import_zod6.z.string()),
-  tags: import_zod6.z.array(import_zod6.z.string()).optional(),
-  displayName: import_zod6.z.string(),
+var ReportCaseSchema = import_zod8.z.object({
+  id: import_zod8.z.string(),
+  runId: import_zod8.z.string(),
+  source: import_zod8.z.string().optional(),
+  file: import_zod8.z.string(),
+  displayFile: import_zod8.z.string(),
+  title: import_zod8.z.string(),
+  fullName: import_zod8.z.string(),
+  ancestorTitles: import_zod8.z.array(import_zod8.z.string()),
+  tags: import_zod8.z.array(import_zod8.z.string()).optional(),
+  displayName: import_zod8.z.string(),
   status: VitestJsonStatusSchema,
   durationMs: FiniteNumberSchema.optional(),
   location: VitestJsonLocationSchema.optional(),
-  failureMessages: import_zod6.z.array(import_zod6.z.string()).default([]),
+  failureMessages: import_zod8.z.array(import_zod8.z.string()).default([]),
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var ReportWorkspaceSchema = import_zod6.z.object({
-  schemaVersion: import_zod6.z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
-  runs: import_zod6.z.array(ReportRunSchema),
-  cases: import_zod6.z.array(ReportCaseSchema)
+var ReportWorkspaceSchema = import_zod8.z.object({
+  schemaVersion: import_zod8.z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
+  runs: import_zod8.z.array(ReportRunSchema),
+  cases: import_zod8.z.array(ReportCaseSchema)
 }).strict();
 function collectReportWorkspace(input, options = {}) {
   const entries = Array.isArray(input) ? input : [input];
@@ -332,6 +344,11 @@ function collectReportWorkspace(input, options = {}) {
         if (!meta) {
           continue;
         }
+        const evalMeta = meta.eval ?? (meta.harness ? {
+          avgScore: assertion.status === "passed" ? 1 : assertion.status === "failed" ? 0 : null,
+          scores: [],
+          thresholdFailed: false
+        } : void 0);
         runCases.push({
           id: createCaseId(runId, file.name, assertion),
           runId,
@@ -347,7 +364,7 @@ function collectReportWorkspace(input, options = {}) {
           ...typeof assertion.duration === "number" ? { durationMs: assertion.duration } : {},
           ...assertion.location ? { location: assertion.location } : {},
           failureMessages: assertion.failureMessages ?? [],
-          ...meta.eval ? { eval: meta.eval } : {},
+          ...evalMeta ? { eval: evalMeta } : {},
           ...meta.harness ? { harness: meta.harness } : {}
         });
       }