@vitest-evals/core 0.16.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +303 -3
- package/dist/index.d.ts +303 -3
- package/dist/index.js +23 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +23 -3
- package/dist/index.mjs.map +1 -1
- package/dist/node.d.mts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +39 -30
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +39 -30
- package/dist/node.mjs.map +1 -1
- package/dist/{workspace-DSZKph1P.d.mts → workspace-BrBLUPEb.d.mts} +200 -2
- package/dist/{workspace-DSZKph1P.d.ts → workspace-BrBLUPEb.d.ts} +200 -2
- package/package.json +1 -1
|
@@ -136,7 +136,7 @@ type VitestJsonReport = z.infer<typeof VitestJsonReportSchema>;
|
|
|
136
136
|
declare function parseVitestJsonReport(input: unknown): VitestJsonReport;
|
|
137
137
|
|
|
138
138
|
/** Current schema version for collected report workspaces. */
|
|
139
|
-
declare const REPORT_WORKSPACE_SCHEMA_VERSION =
|
|
139
|
+
declare const REPORT_WORKSPACE_SCHEMA_VERSION = 2;
|
|
140
140
|
/** One collected Vitest JSON report source in a multi-run workspace. */
|
|
141
141
|
declare const ReportRunSchema: z.ZodObject<{
|
|
142
142
|
id: z.ZodString;
|
|
@@ -189,6 +189,104 @@ declare const ReportCaseSchema: z.ZodObject<{
|
|
|
189
189
|
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
190
190
|
name: z.ZodOptional<z.ZodString>;
|
|
191
191
|
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
192
|
+
judgeRuns: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
193
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
194
|
+
session: z.ZodObject<{
|
|
195
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
196
|
+
type: z.ZodLiteral<"message">;
|
|
197
|
+
role: z.ZodEnum<{
|
|
198
|
+
system: "system";
|
|
199
|
+
user: "user";
|
|
200
|
+
assistant: "assistant";
|
|
201
|
+
}>;
|
|
202
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
203
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
204
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
205
|
+
type: z.ZodLiteral<"tool_call">;
|
|
206
|
+
id: z.ZodString;
|
|
207
|
+
name: z.ZodString;
|
|
208
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
209
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
210
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
211
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
212
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
213
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
214
|
+
type: z.ZodLiteral<"tool_result">;
|
|
215
|
+
toolCallId: z.ZodString;
|
|
216
|
+
name: z.ZodOptional<z.ZodString>;
|
|
217
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
218
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
219
|
+
message: z.ZodString;
|
|
220
|
+
type: z.ZodOptional<z.ZodString>;
|
|
221
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
222
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
223
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
224
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
225
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
226
|
+
}, z.core.$strict>], "type">>;
|
|
227
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
228
|
+
model: z.ZodOptional<z.ZodString>;
|
|
229
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
230
|
+
}, z.core.$strict>;
|
|
231
|
+
usage: z.ZodObject<{
|
|
232
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
233
|
+
model: z.ZodOptional<z.ZodString>;
|
|
234
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
235
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
236
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
237
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
238
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
239
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
240
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
241
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
242
|
+
}, z.core.$strict>;
|
|
243
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
244
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
245
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
246
|
+
}, z.core.$strict>>;
|
|
247
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
248
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
249
|
+
id: z.ZodOptional<z.ZodString>;
|
|
250
|
+
name: z.ZodOptional<z.ZodString>;
|
|
251
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
252
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
253
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
254
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
255
|
+
spans: z.ZodArray<z.ZodObject<{
|
|
256
|
+
id: z.ZodOptional<z.ZodString>;
|
|
257
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
258
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
259
|
+
name: z.ZodString;
|
|
260
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
261
|
+
custom: "custom";
|
|
262
|
+
tool: "tool";
|
|
263
|
+
model: "model";
|
|
264
|
+
run: "run";
|
|
265
|
+
agent: "agent";
|
|
266
|
+
guardrail: "guardrail";
|
|
267
|
+
handoff: "handoff";
|
|
268
|
+
}>>;
|
|
269
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
270
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
271
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
272
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
273
|
+
error: "error";
|
|
274
|
+
ok: "ok";
|
|
275
|
+
}>>;
|
|
276
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
277
|
+
message: z.ZodString;
|
|
278
|
+
type: z.ZodOptional<z.ZodString>;
|
|
279
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
280
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
281
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
282
|
+
name: z.ZodString;
|
|
283
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
284
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
285
|
+
}, z.core.$strict>>>;
|
|
286
|
+
}, z.core.$strict>>;
|
|
287
|
+
}, z.core.$strict>>>;
|
|
288
|
+
errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
289
|
+
}, z.core.$strict>>>;
|
|
192
290
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
193
291
|
}, z.core.$strict>>>;
|
|
194
292
|
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
@@ -261,6 +359,7 @@ declare const ReportCaseSchema: z.ZodObject<{
|
|
|
261
359
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
262
360
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
263
361
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
362
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
264
363
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
265
364
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
266
365
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
@@ -318,7 +417,6 @@ declare const ReportCaseSchema: z.ZodObject<{
|
|
|
318
417
|
type ReportCase = z.infer<typeof ReportCaseSchema>;
|
|
319
418
|
/** Full multi-run report workspace consumed by rich report UIs. */
|
|
320
419
|
declare const ReportWorkspaceSchema: z.ZodObject<{
|
|
321
|
-
schemaVersion: z.ZodLiteral<1>;
|
|
322
420
|
runs: z.ZodArray<z.ZodObject<{
|
|
323
421
|
id: z.ZodString;
|
|
324
422
|
source: z.ZodOptional<z.ZodString>;
|
|
@@ -367,6 +465,104 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
|
|
|
367
465
|
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
368
466
|
name: z.ZodOptional<z.ZodString>;
|
|
369
467
|
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
468
|
+
judgeRuns: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
469
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
470
|
+
session: z.ZodObject<{
|
|
471
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
472
|
+
type: z.ZodLiteral<"message">;
|
|
473
|
+
role: z.ZodEnum<{
|
|
474
|
+
system: "system";
|
|
475
|
+
user: "user";
|
|
476
|
+
assistant: "assistant";
|
|
477
|
+
}>;
|
|
478
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
479
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
480
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
481
|
+
type: z.ZodLiteral<"tool_call">;
|
|
482
|
+
id: z.ZodString;
|
|
483
|
+
name: z.ZodString;
|
|
484
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
485
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
486
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
487
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
488
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
489
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
490
|
+
type: z.ZodLiteral<"tool_result">;
|
|
491
|
+
toolCallId: z.ZodString;
|
|
492
|
+
name: z.ZodOptional<z.ZodString>;
|
|
493
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
494
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
495
|
+
message: z.ZodString;
|
|
496
|
+
type: z.ZodOptional<z.ZodString>;
|
|
497
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
498
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
499
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
500
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
501
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
502
|
+
}, z.core.$strict>], "type">>;
|
|
503
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
504
|
+
model: z.ZodOptional<z.ZodString>;
|
|
505
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
506
|
+
}, z.core.$strict>;
|
|
507
|
+
usage: z.ZodObject<{
|
|
508
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
509
|
+
model: z.ZodOptional<z.ZodString>;
|
|
510
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
511
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
512
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
513
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
514
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
515
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
516
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
517
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
518
|
+
}, z.core.$strict>;
|
|
519
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
520
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
521
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
522
|
+
}, z.core.$strict>>;
|
|
523
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
524
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
525
|
+
id: z.ZodOptional<z.ZodString>;
|
|
526
|
+
name: z.ZodOptional<z.ZodString>;
|
|
527
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
528
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
529
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
530
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
531
|
+
spans: z.ZodArray<z.ZodObject<{
|
|
532
|
+
id: z.ZodOptional<z.ZodString>;
|
|
533
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
534
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
535
|
+
name: z.ZodString;
|
|
536
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
537
|
+
custom: "custom";
|
|
538
|
+
tool: "tool";
|
|
539
|
+
model: "model";
|
|
540
|
+
run: "run";
|
|
541
|
+
agent: "agent";
|
|
542
|
+
guardrail: "guardrail";
|
|
543
|
+
handoff: "handoff";
|
|
544
|
+
}>>;
|
|
545
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
546
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
547
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
548
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
549
|
+
error: "error";
|
|
550
|
+
ok: "ok";
|
|
551
|
+
}>>;
|
|
552
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
553
|
+
message: z.ZodString;
|
|
554
|
+
type: z.ZodOptional<z.ZodString>;
|
|
555
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
556
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
557
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
558
|
+
name: z.ZodString;
|
|
559
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
560
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
561
|
+
}, z.core.$strict>>>;
|
|
562
|
+
}, z.core.$strict>>;
|
|
563
|
+
}, z.core.$strict>>>;
|
|
564
|
+
errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
565
|
+
}, z.core.$strict>>>;
|
|
370
566
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
371
567
|
}, z.core.$strict>>>;
|
|
372
568
|
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
@@ -439,6 +635,7 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
|
|
|
439
635
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
440
636
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
441
637
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
638
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
442
639
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
443
640
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
444
641
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
@@ -492,6 +689,7 @@ declare const ReportWorkspaceSchema: z.ZodObject<{
|
|
|
492
689
|
}, z.core.$strict>>;
|
|
493
690
|
}, z.core.$strict>>;
|
|
494
691
|
}, z.core.$strict>>;
|
|
692
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
495
693
|
}, z.core.$strict>;
|
|
496
694
|
/** Full multi-run report workspace consumed by rich report UIs. */
|
|
497
695
|
type ReportWorkspace = z.infer<typeof ReportWorkspaceSchema>;
|