@vitest-evals/core 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,10 +15,10 @@ declare const JsonObjectSchema: z.ZodRecord<z.ZodString, z.ZodType<JsonValue, un
15
15
 
16
16
  /** Status values emitted by Vitest JSON reports. */
17
17
  declare const VitestJsonStatusSchema: z.ZodEnum<{
18
+ pending: "pending";
18
19
  passed: "passed";
19
20
  failed: "failed";
20
21
  skipped: "skipped";
21
- pending: "pending";
22
22
  todo: "todo";
23
23
  disabled: "disabled";
24
24
  }>;
@@ -36,10 +36,10 @@ declare const VitestJsonAssertionSchema: z.ZodObject<{
36
36
  ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
37
37
  fullName: z.ZodString;
38
38
  status: z.ZodEnum<{
39
+ pending: "pending";
39
40
  passed: "passed";
40
41
  failed: "failed";
41
42
  skipped: "skipped";
42
- pending: "pending";
43
43
  todo: "todo";
44
44
  disabled: "disabled";
45
45
  }>;
@@ -69,10 +69,10 @@ declare const VitestJsonFileSchema: z.ZodObject<{
69
69
  ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
70
70
  fullName: z.ZodString;
71
71
  status: z.ZodEnum<{
72
+ pending: "pending";
72
73
  passed: "passed";
73
74
  failed: "failed";
74
75
  skipped: "skipped";
75
- pending: "pending";
76
76
  todo: "todo";
77
77
  disabled: "disabled";
78
78
  }>;
@@ -111,10 +111,10 @@ declare const VitestJsonReportSchema: z.ZodObject<{
111
111
  ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
112
112
  fullName: z.ZodString;
113
113
  status: z.ZodEnum<{
114
+ pending: "pending";
114
115
  passed: "passed";
115
116
  failed: "failed";
116
117
  skipped: "skipped";
117
- pending: "pending";
118
118
  todo: "todo";
119
119
  disabled: "disabled";
120
120
  }>;
@@ -172,10 +172,10 @@ declare const ReportCaseSchema: z.ZodObject<{
172
172
  tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
173
173
  displayName: z.ZodString;
174
174
  status: z.ZodEnum<{
175
+ pending: "pending";
175
176
  passed: "passed";
176
177
  failed: "failed";
177
178
  skipped: "skipped";
178
- pending: "pending";
179
179
  todo: "todo";
180
180
  disabled: "disabled";
181
181
  }>;
@@ -194,50 +194,62 @@ declare const ReportCaseSchema: z.ZodObject<{
194
194
  avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
195
195
  output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
196
196
  thresholdFailed: z.ZodOptional<z.ZodBoolean>;
197
- toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
198
- id: z.ZodOptional<z.ZodString>;
197
+ toolCalls: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
199
198
  name: z.ZodString;
200
199
  arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
200
+ status: z.ZodLiteral<"pending">;
201
+ }, z.core.$strict>, z.ZodObject<{
202
+ name: z.ZodString;
203
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
204
+ status: z.ZodLiteral<"ok">;
201
205
  result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
202
- error: z.ZodOptional<z.ZodObject<{
206
+ }, z.core.$strict>, z.ZodObject<{
207
+ name: z.ZodString;
208
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
209
+ status: z.ZodLiteral<"error">;
210
+ error: z.ZodObject<{
203
211
  message: z.ZodString;
204
212
  type: z.ZodOptional<z.ZodString>;
205
- }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
206
- startedAt: z.ZodOptional<z.ZodString>;
207
- finishedAt: z.ZodOptional<z.ZodString>;
208
- durationMs: z.ZodOptional<z.ZodNumber>;
209
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
210
- }, z.core.$strict>>>;
213
+ }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
214
+ }, z.core.$strict>], "status">>>;
211
215
  }, z.core.$strict>>;
212
216
  harness: z.ZodOptional<z.ZodObject<{
213
217
  name: z.ZodOptional<z.ZodString>;
214
218
  run: z.ZodOptional<z.ZodObject<{
215
219
  output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
216
220
  session: z.ZodObject<{
217
- messages: z.ZodDefault<z.ZodArray<z.ZodObject<{
221
+ events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
222
+ type: z.ZodLiteral<"message">;
218
223
  role: z.ZodEnum<{
219
224
  system: "system";
220
225
  user: "user";
221
226
  assistant: "assistant";
222
- tool: "tool";
223
227
  }>;
224
228
  content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
225
- toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
226
- id: z.ZodOptional<z.ZodString>;
227
- name: z.ZodString;
228
- arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
229
- result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
230
- error: z.ZodOptional<z.ZodObject<{
231
- message: z.ZodString;
232
- type: z.ZodOptional<z.ZodString>;
233
- }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
234
- startedAt: z.ZodOptional<z.ZodString>;
235
- finishedAt: z.ZodOptional<z.ZodString>;
236
- durationMs: z.ZodOptional<z.ZodNumber>;
237
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
238
- }, z.core.$strict>>>;
239
229
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
240
- }, z.core.$strict>>>;
230
+ }, z.core.$strict>, z.ZodObject<{
231
+ type: z.ZodLiteral<"tool_call">;
232
+ id: z.ZodString;
233
+ name: z.ZodString;
234
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
235
+ startedAt: z.ZodOptional<z.ZodString>;
236
+ finishedAt: z.ZodOptional<z.ZodString>;
237
+ durationMs: z.ZodOptional<z.ZodNumber>;
238
+ metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
239
+ }, z.core.$strict>, z.ZodObject<{
240
+ type: z.ZodLiteral<"tool_result">;
241
+ toolCallId: z.ZodString;
242
+ name: z.ZodOptional<z.ZodString>;
243
+ content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
244
+ error: z.ZodOptional<z.ZodObject<{
245
+ message: z.ZodString;
246
+ type: z.ZodOptional<z.ZodString>;
247
+ }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
248
+ startedAt: z.ZodOptional<z.ZodString>;
249
+ finishedAt: z.ZodOptional<z.ZodString>;
250
+ durationMs: z.ZodOptional<z.ZodNumber>;
251
+ metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
252
+ }, z.core.$strict>], "type">>;
241
253
  provider: z.ZodOptional<z.ZodString>;
242
254
  model: z.ZodOptional<z.ZodString>;
243
255
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
@@ -265,15 +277,15 @@ declare const ReportCaseSchema: z.ZodObject<{
265
277
  finishedAt: z.ZodOptional<z.ZodString>;
266
278
  durationMs: z.ZodOptional<z.ZodNumber>;
267
279
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
268
- spans: z.ZodDefault<z.ZodArray<z.ZodObject<{
280
+ spans: z.ZodArray<z.ZodObject<{
269
281
  id: z.ZodOptional<z.ZodString>;
270
282
  traceId: z.ZodOptional<z.ZodString>;
271
283
  parentId: z.ZodOptional<z.ZodString>;
272
284
  name: z.ZodString;
273
285
  kind: z.ZodOptional<z.ZodEnum<{
274
286
  custom: "custom";
275
- model: "model";
276
287
  tool: "tool";
288
+ model: "model";
277
289
  run: "run";
278
290
  agent: "agent";
279
291
  guardrail: "guardrail";
@@ -296,9 +308,9 @@ declare const ReportCaseSchema: z.ZodObject<{
296
308
  timestamp: z.ZodOptional<z.ZodString>;
297
309
  attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
298
310
  }, z.core.$strict>>>;
299
- }, z.core.$strict>>>;
311
+ }, z.core.$strict>>;
300
312
  }, z.core.$strict>>>;
301
- errors: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
313
+ errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
302
314
  }, z.core.$strict>>;
303
315
  }, z.core.$strict>>;
304
316
  }, z.core.$strict>;
@@ -338,10 +350,10 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
338
350
  tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
339
351
  displayName: z.ZodString;
340
352
  status: z.ZodEnum<{
353
+ pending: "pending";
341
354
  passed: "passed";
342
355
  failed: "failed";
343
356
  skipped: "skipped";
344
- pending: "pending";
345
357
  todo: "todo";
346
358
  disabled: "disabled";
347
359
  }>;
@@ -360,50 +372,62 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
360
372
  avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
361
373
  output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
362
374
  thresholdFailed: z.ZodOptional<z.ZodBoolean>;
363
- toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
364
- id: z.ZodOptional<z.ZodString>;
375
+ toolCalls: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
365
376
  name: z.ZodString;
366
377
  arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
378
+ status: z.ZodLiteral<"pending">;
379
+ }, z.core.$strict>, z.ZodObject<{
380
+ name: z.ZodString;
381
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
382
+ status: z.ZodLiteral<"ok">;
367
383
  result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
368
- error: z.ZodOptional<z.ZodObject<{
384
+ }, z.core.$strict>, z.ZodObject<{
385
+ name: z.ZodString;
386
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
387
+ status: z.ZodLiteral<"error">;
388
+ error: z.ZodObject<{
369
389
  message: z.ZodString;
370
390
  type: z.ZodOptional<z.ZodString>;
371
- }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
372
- startedAt: z.ZodOptional<z.ZodString>;
373
- finishedAt: z.ZodOptional<z.ZodString>;
374
- durationMs: z.ZodOptional<z.ZodNumber>;
375
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
376
- }, z.core.$strict>>>;
391
+ }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
392
+ }, z.core.$strict>], "status">>>;
377
393
  }, z.core.$strict>>;
378
394
  harness: z.ZodOptional<z.ZodObject<{
379
395
  name: z.ZodOptional<z.ZodString>;
380
396
  run: z.ZodOptional<z.ZodObject<{
381
397
  output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
382
398
  session: z.ZodObject<{
383
- messages: z.ZodDefault<z.ZodArray<z.ZodObject<{
399
+ events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
400
+ type: z.ZodLiteral<"message">;
384
401
  role: z.ZodEnum<{
385
402
  system: "system";
386
403
  user: "user";
387
404
  assistant: "assistant";
388
- tool: "tool";
389
405
  }>;
390
406
  content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
391
- toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
392
- id: z.ZodOptional<z.ZodString>;
393
- name: z.ZodString;
394
- arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
395
- result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
396
- error: z.ZodOptional<z.ZodObject<{
397
- message: z.ZodString;
398
- type: z.ZodOptional<z.ZodString>;
399
- }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
400
- startedAt: z.ZodOptional<z.ZodString>;
401
- finishedAt: z.ZodOptional<z.ZodString>;
402
- durationMs: z.ZodOptional<z.ZodNumber>;
403
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
404
- }, z.core.$strict>>>;
405
407
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
406
- }, z.core.$strict>>>;
408
+ }, z.core.$strict>, z.ZodObject<{
409
+ type: z.ZodLiteral<"tool_call">;
410
+ id: z.ZodString;
411
+ name: z.ZodString;
412
+ arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
413
+ startedAt: z.ZodOptional<z.ZodString>;
414
+ finishedAt: z.ZodOptional<z.ZodString>;
415
+ durationMs: z.ZodOptional<z.ZodNumber>;
416
+ metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
417
+ }, z.core.$strict>, z.ZodObject<{
418
+ type: z.ZodLiteral<"tool_result">;
419
+ toolCallId: z.ZodString;
420
+ name: z.ZodOptional<z.ZodString>;
421
+ content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
422
+ error: z.ZodOptional<z.ZodObject<{
423
+ message: z.ZodString;
424
+ type: z.ZodOptional<z.ZodString>;
425
+ }, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
426
+ startedAt: z.ZodOptional<z.ZodString>;
427
+ finishedAt: z.ZodOptional<z.ZodString>;
428
+ durationMs: z.ZodOptional<z.ZodNumber>;
429
+ metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
430
+ }, z.core.$strict>], "type">>;
407
431
  provider: z.ZodOptional<z.ZodString>;
408
432
  model: z.ZodOptional<z.ZodString>;
409
433
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
@@ -431,15 +455,15 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
431
455
  finishedAt: z.ZodOptional<z.ZodString>;
432
456
  durationMs: z.ZodOptional<z.ZodNumber>;
433
457
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
434
- spans: z.ZodDefault<z.ZodArray<z.ZodObject<{
458
+ spans: z.ZodArray<z.ZodObject<{
435
459
  id: z.ZodOptional<z.ZodString>;
436
460
  traceId: z.ZodOptional<z.ZodString>;
437
461
  parentId: z.ZodOptional<z.ZodString>;
438
462
  name: z.ZodString;
439
463
  kind: z.ZodOptional<z.ZodEnum<{
440
464
  custom: "custom";
441
- model: "model";
442
465
  tool: "tool";
466
+ model: "model";
443
467
  run: "run";
444
468
  agent: "agent";
445
469
  guardrail: "guardrail";
@@ -462,9 +486,9 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
462
486
  timestamp: z.ZodOptional<z.ZodString>;
463
487
  attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
464
488
  }, z.core.$strict>>>;
465
- }, z.core.$strict>>>;
489
+ }, z.core.$strict>>;
466
490
  }, z.core.$strict>>>;
467
- errors: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
491
+ errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
468
492
  }, z.core.$strict>>;
469
493
  }, z.core.$strict>>;
470
494
  }, z.core.$strict>>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vitest-evals/core",
3
- "version": "0.13.0",
3
+ "version": "0.14.0",
4
4
  "description": "Shared primitives, schemas, and report collection helpers for vitest-evals.",
5
5
  "repository": {
6
6
  "type": "git",