@vitest-evals/core 0.13.1 → 0.14.0

package/dist/node.mjs

@@ -3,13 +3,13 @@ import { readdir, readFile, stat } from "fs/promises";
 import { extname, isAbsolute, relative, resolve, sep } from "path";
 
 // src/report/workspace.ts
-import { z as z6 } from "zod";
+import { z as z8 } from "zod";
 
 // src/report/metadata.ts
-import { z as z4 } from "zod";
+import { z as z6 } from "zod";
 
 // src/harness/index.ts
-import { z as z3 } from "zod";
+import { z as z5 } from "zod";
 
 // src/json.ts
 import { z } from "zod";
@@ -57,10 +57,52 @@ function isRecord(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
 
+// src/harness/errors.ts
+import { z as z3 } from "zod";
+var NormalizedErrorSchema = z3.object({
+  message: z3.string(),
+  type: z3.string().optional()
+}).catchall(JsonValueSchema);
+
+// src/harness/transcript.ts
+import { z as z4 } from "zod";
+var TranscriptToolCallEventSchema = z4.object({
+  type: z4.literal("tool_call"),
+  id: z4.string(),
+  name: z4.string(),
+  arguments: JsonObjectSchema.optional(),
+  startedAt: z4.string().optional(),
+  finishedAt: z4.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptMessageEventSchema = z4.object({
+  type: z4.literal("message"),
+  role: z4.enum(["system", "user", "assistant"]),
+  content: JsonValueSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptToolResultEventSchema = z4.object({
+  type: z4.literal("tool_result"),
+  toolCallId: z4.string(),
+  name: z4.string().optional(),
+  content: JsonValueSchema.optional(),
+  error: NormalizedErrorSchema.optional(),
+  startedAt: z4.string().optional(),
+  finishedAt: z4.string().optional(),
+  durationMs: FiniteNumberSchema.optional(),
+  metadata: JsonObjectSchema.optional()
+}).strict();
+var TranscriptEventSchema = z4.discriminatedUnion("type", [
+  TranscriptMessageEventSchema,
+  TranscriptToolCallEventSchema,
+  TranscriptToolResultEventSchema
+]);
+
 // src/harness/index.ts
-var UsageSummarySchema = z3.object({
-  provider: z3.string().optional(),
-  model: z3.string().optional(),
+var UsageSummarySchema = z5.object({
+  provider: z5.string().optional(),
+  model: z5.string().optional(),
   inputTokens: FiniteNumberSchema.optional(),
   outputTokens: FiniteNumberSchema.optional(),
   reasoningTokens: FiniteNumberSchema.optional(),
@@ -69,141 +111,111 @@ var UsageSummarySchema = z3.object({
   retries: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var TimingSummarySchema = z3.object({
+var TimingSummarySchema = z5.object({
   totalMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
-var NormalizedErrorSchema = z3.object({
-  message: z3.string(),
-  type: z3.string().optional()
-}).catchall(JsonValueSchema);
-var ToolCallRecordSchema = z3.object({
-  id: z3.string().optional(),
-  name: z3.string(),
-  arguments: JsonObjectSchema.optional(),
-  result: JsonValueSchema.optional(),
-  error: NormalizedErrorSchema.optional(),
-  startedAt: z3.string().optional(),
-  finishedAt: z3.string().optional(),
-  durationMs: FiniteNumberSchema.optional(),
-  metadata: JsonObjectSchema.optional()
-}).strict();
-var NormalizedMessageSchema = z3.object({
-  role: z3.enum(["system", "user", "assistant", "tool"]),
-  content: JsonValueSchema.optional(),
-  toolCalls: z3.array(ToolCallRecordSchema).optional(),
-  metadata: JsonObjectSchema.optional()
+var ToolCallBaseSchema = z5.object({
+  name: z5.string(),
+  arguments: JsonObjectSchema.optional()
 }).strict();
-var NormalizedSessionSchema = z3.object({
-  messages: z3.array(NormalizedMessageSchema).default([]),
-  provider: z3.string().optional(),
-  model: z3.string().optional(),
+var ToolCallSchema = z5.discriminatedUnion("status", [
+  ToolCallBaseSchema.extend({
+    status: z5.literal("pending")
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: z5.literal("ok"),
+    result: JsonValueSchema.optional()
+  }).strict(),
+  ToolCallBaseSchema.extend({
+    status: z5.literal("error"),
+    error: NormalizedErrorSchema
+  }).strict()
+]);
+var NormalizedSessionSchema = z5.object({
+  events: z5.array(TranscriptEventSchema),
+  provider: z5.string().optional(),
+  model: z5.string().optional(),
   metadata: JsonObjectSchema.optional()
 }).strict();
 var NormalizedSpanAttributesSchema = JsonObjectSchema;
-var NormalizedSpanEventSchema = z3.object({
-  name: z3.string(),
-  timestamp: z3.string().optional(),
+var NormalizedSpanEventSchema = z5.object({
+  name: z5.string(),
+  timestamp: z5.string().optional(),
   attributes: NormalizedSpanAttributesSchema.optional()
 }).strict();
-var NormalizedSpanSchema = z3.object({
-  id: z3.string().optional(),
-  traceId: z3.string().optional(),
-  parentId: z3.string().optional(),
-  name: z3.string(),
-  kind: z3.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
-  startedAt: z3.string().optional(),
-  finishedAt: z3.string().optional(),
+var NormalizedSpanSchema = z5.object({
+  id: z5.string().optional(),
+  traceId: z5.string().optional(),
+  parentId: z5.string().optional(),
+  name: z5.string(),
+  kind: z5.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
+  startedAt: z5.string().optional(),
+  finishedAt: z5.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
-  status: z3.enum(["ok", "error"]).optional(),
+  status: z5.enum(["ok", "error"]).optional(),
   error: NormalizedErrorSchema.optional(),
   attributes: NormalizedSpanAttributesSchema.optional(),
-  events: z3.array(NormalizedSpanEventSchema).optional()
+  events: z5.array(NormalizedSpanEventSchema).optional()
 }).strict();
-var NormalizedTraceSchema = z3.object({
-  id: z3.string().optional(),
-  name: z3.string().optional(),
-  startedAt: z3.string().optional(),
-  finishedAt: z3.string().optional(),
+var NormalizedTraceSchema = z5.object({
+  id: z5.string().optional(),
+  name: z5.string().optional(),
+  startedAt: z5.string().optional(),
+  finishedAt: z5.string().optional(),
   durationMs: FiniteNumberSchema.optional(),
   metadata: JsonObjectSchema.optional(),
-  spans: z3.array(NormalizedSpanSchema).default([])
+  spans: z5.array(NormalizedSpanSchema)
 }).strict();
-var HarnessRunSchema = z3.object({
+var HarnessRunSchema = z5.object({
   output: JsonValueSchema.optional(),
   session: NormalizedSessionSchema,
   usage: UsageSummarySchema,
   timings: TimingSummarySchema.optional(),
   artifacts: JsonObjectSchema.optional(),
-  traces: z3.array(NormalizedTraceSchema).optional(),
-  errors: z3.array(JsonObjectSchema).default([])
+  traces: z5.array(NormalizedTraceSchema).optional(),
+  errors: z5.array(JsonObjectSchema)
 }).strict();
 
 // src/report/metadata.ts
-var HarnessMetaSchema = z4.object({
-  name: z4.string().optional(),
+var HarnessMetaSchema = z6.object({
+  name: z6.string().optional(),
   run: HarnessRunSchema.optional()
 }).strict();
-var EvalScoreSchema = z4.object({
-  name: z4.string().optional(),
+var EvalScoreSchema = z6.object({
+  name: z6.string().optional(),
   score: NullableFiniteNumberSchema,
   metadata: JsonObjectSchema.optional()
 }).strict();
-var EvalMetaSchema = z4.object({
-  scores: z4.array(EvalScoreSchema).optional(),
+var EvalMetaSchema = z6.object({
+  scores: z6.array(EvalScoreSchema).optional(),
   avgScore: NullableFiniteNumberSchema,
   output: JsonValueSchema.optional(),
-  thresholdFailed: z4.boolean().optional(),
-  toolCalls: z4.array(ToolCallRecordSchema).optional()
+  thresholdFailed: z6.boolean().optional(),
+  toolCalls: z6.array(ToolCallSchema).optional()
 }).strict();
-var EvalTaskMetaSchema = z4.object({
+var EvalTaskMetaSchema = z6.object({
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var LenientToolCallRecordSchema = ToolCallRecordSchema.strip();
-var LenientMessageSchema = NormalizedMessageSchema.extend({
-  toolCalls: z4.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
-var LenientSessionSchema = NormalizedSessionSchema.extend({
-  messages: z4.array(LenientMessageSchema).default([]).catch([])
-}).strip();
-var LenientSpanEventSchema = NormalizedSpanEventSchema.strip();
-var LenientSpanSchema = NormalizedSpanSchema.extend({
-  events: z4.array(LenientSpanEventSchema).optional().catch(void 0)
-}).strip();
-var LenientTraceSchema = NormalizedTraceSchema.extend({
-  spans: z4.array(LenientSpanSchema).default([]).catch([])
-}).strip();
-var LenientHarnessRunSchema = HarnessRunSchema.extend({
-  session: LenientSessionSchema,
-  usage: UsageSummarySchema.strip().default({}),
-  timings: TimingSummarySchema.strip().optional(),
-  traces: z4.array(LenientTraceSchema).optional().catch(void 0)
-}).strip();
-var LenientHarnessMetaSchema = HarnessMetaSchema.extend({
-  run: LenientHarnessRunSchema.optional().catch(void 0)
-}).strip();
-var LenientEvalScoreSchema = EvalScoreSchema.strip();
-var LenientEvalMetaSchema = EvalMetaSchema.extend({
-  scores: z4.array(LenientEvalScoreSchema).optional().catch(void 0),
-  toolCalls: z4.array(LenientToolCallRecordSchema).optional().catch(void 0)
-}).strip();
 function readEvalTaskMeta(input) {
   if (!isJsonObject(input)) {
     return void 0;
   }
-  const evalResult = LenientEvalMetaSchema.safeParse(input.eval);
-  const harnessResult = LenientHarnessMetaSchema.safeParse(input.harness);
   const meta = {
-    ...evalResult.success && input.eval !== void 0 ? { eval: evalResult.data } : {},
-    ...harnessResult.success && input.harness !== void 0 ? { harness: harnessResult.data } : {}
+    ...input.eval !== void 0 ? { eval: input.eval } : {},
+    ...input.harness !== void 0 ? { harness: input.harness } : {}
   };
-  return meta.eval || meta.harness ? meta : void 0;
+  if (!("eval" in meta) && !("harness" in meta)) {
+    return void 0;
+  }
+  const result = EvalTaskMetaSchema.safeParse(meta);
+  return result.success ? result.data : void 0;
 }
 
 // src/report/vitest-json.ts
-import { z as z5 } from "zod";
-var VitestJsonStatusSchema = z5.enum([
+import { z as z7 } from "zod";
+var VitestJsonStatusSchema = z7.enum([
   "passed",
   "failed",
   "skipped",
@@ -211,38 +223,38 @@ var VitestJsonStatusSchema = z5.enum([
   "todo",
   "disabled"
 ]);
-var VitestJsonLocationSchema = z5.object({
+var VitestJsonLocationSchema = z7.object({
   line: FiniteNumberSchema,
   column: FiniteNumberSchema
 }).passthrough();
-var VitestJsonAssertionSchema = z5.object({
-  ancestorTitles: z5.array(z5.string()).default([]),
-  fullName: z5.string(),
+var VitestJsonAssertionSchema = z7.object({
+  ancestorTitles: z7.array(z7.string()).default([]),
+  fullName: z7.string(),
   status: VitestJsonStatusSchema,
-  title: z5.string(),
-  meta: z5.unknown().optional(),
+  title: z7.string(),
+  meta: z7.unknown().optional(),
   duration: FiniteNumberSchema.nullable().optional(),
-  failureMessages: z5.array(z5.string()).nullable().optional(),
+  failureMessages: z7.array(z7.string()).nullable().optional(),
   location: VitestJsonLocationSchema.nullable().optional(),
-  tags: z5.array(z5.string()).optional()
+  tags: z7.array(z7.string()).optional()
 }).passthrough();
-var VitestJsonFileSchema = z5.object({
-  message: z5.string(),
-  name: z5.string(),
-  status: z5.enum(["failed", "passed"]),
+var VitestJsonFileSchema = z7.object({
+  message: z7.string(),
+  name: z7.string(),
+  status: z7.enum(["failed", "passed"]),
   startTime: OptionalFiniteNumberSchema,
   endTime: OptionalFiniteNumberSchema,
-  assertionResults: z5.array(VitestJsonAssertionSchema).default([])
+  assertionResults: z7.array(VitestJsonAssertionSchema).default([])
 }).passthrough();
-var VitestJsonReportSchema = z5.object({
+var VitestJsonReportSchema = z7.object({
   numFailedTests: FiniteNumberSchema,
   numPassedTests: FiniteNumberSchema,
   numPendingTests: FiniteNumberSchema,
   numTodoTests: FiniteNumberSchema,
   numTotalTests: FiniteNumberSchema,
   startTime: FiniteNumberSchema,
-  success: z5.boolean(),
-  testResults: z5.array(VitestJsonFileSchema).default([])
+  success: z7.boolean(),
+  testResults: z7.array(VitestJsonFileSchema).default([])
 }).passthrough();
 function parseVitestJsonReport(input) {
   return parseWithSchema(VitestJsonReportSchema, input, "Vitest JSON report");
@@ -250,13 +262,13 @@ function parseVitestJsonReport(input) {
 
 // src/report/workspace.ts
 var REPORT_WORKSPACE_SCHEMA_VERSION = 1;
-var ReportRunSchema = z6.object({
-  id: z6.string(),
-  source: z6.string().optional(),
-  status: z6.enum(["passed", "failed"]),
+var ReportRunSchema = z8.object({
+  id: z8.string(),
+  source: z8.string().optional(),
+  status: z8.enum(["passed", "failed"]),
   startedAt: FiniteNumberSchema.optional(),
   durationMs: FiniteNumberSchema.optional(),
-  totals: z6.object({
+  totals: z8.object({
     total: FiniteNumberSchema,
     passed: FiniteNumberSchema,
     failed: FiniteNumberSchema,
@@ -266,28 +278,28 @@ var ReportRunSchema = z6.object({
     evalFailed: FiniteNumberSchema
   })
 }).strict();
-var ReportCaseSchema = z6.object({
-  id: z6.string(),
-  runId: z6.string(),
-  source: z6.string().optional(),
-  file: z6.string(),
-  displayFile: z6.string(),
-  title: z6.string(),
-  fullName: z6.string(),
-  ancestorTitles: z6.array(z6.string()),
-  tags: z6.array(z6.string()).optional(),
-  displayName: z6.string(),
+var ReportCaseSchema = z8.object({
+  id: z8.string(),
+  runId: z8.string(),
+  source: z8.string().optional(),
+  file: z8.string(),
+  displayFile: z8.string(),
+  title: z8.string(),
+  fullName: z8.string(),
+  ancestorTitles: z8.array(z8.string()),
+  tags: z8.array(z8.string()).optional(),
+  displayName: z8.string(),
   status: VitestJsonStatusSchema,
   durationMs: FiniteNumberSchema.optional(),
   location: VitestJsonLocationSchema.optional(),
-  failureMessages: z6.array(z6.string()).default([]),
+  failureMessages: z8.array(z8.string()).default([]),
   eval: EvalMetaSchema.optional(),
   harness: HarnessMetaSchema.optional()
 }).strict();
-var ReportWorkspaceSchema = z6.object({
-  schemaVersion: z6.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
-  runs: z6.array(ReportRunSchema),
-  cases: z6.array(ReportCaseSchema)
+var ReportWorkspaceSchema = z8.object({
+  schemaVersion: z8.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
+  runs: z8.array(ReportRunSchema),
+  cases: z8.array(ReportCaseSchema)
 }).strict();
 function collectReportWorkspace(input, options = {}) {
   const entries = Array.isArray(input) ? input : [input];