@vitest-evals/core 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +36 -0
- package/dist/index.d.mts +895 -0
- package/dist/index.d.ts +895 -0
- package/dist/index.js +557 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +489 -0
- package/dist/index.mjs.map +1 -0
- package/dist/node.d.mts +33 -0
- package/dist/node.d.ts +33 -0
- package/dist/node.js +628 -0
- package/dist/node.js.map +1 -0
- package/dist/node.mjs +598 -0
- package/dist/node.mjs.map +1 -0
- package/dist/workspace-BNgjyW7W.d.mts +489 -0
- package/dist/workspace-BNgjyW7W.d.ts +489 -0
- package/package.json +46 -0
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
/** Primitive scalar values allowed in persisted vitest-evals JSON artifacts. */
|
|
4
|
+
type JsonPrimitive = string | number | boolean | null;
|
|
5
|
+
/** JSON-safe value shape used by reports, normalized sessions, and traces. */
|
|
6
|
+
type JsonValue = JsonPrimitive | JsonValue[] | {
|
|
7
|
+
[key: string]: JsonValue;
|
|
8
|
+
};
|
|
9
|
+
/** Schema for primitive scalar values in persisted report artifacts. */
|
|
10
|
+
declare const JsonPrimitiveSchema: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>;
|
|
11
|
+
/** Schema for any JSON-safe value in persisted report artifacts. */
|
|
12
|
+
declare const JsonValueSchema: z.ZodType<JsonValue>;
|
|
13
|
+
/** Schema for JSON-safe object records in persisted report artifacts. */
|
|
14
|
+
declare const JsonObjectSchema: z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
15
|
+
|
|
16
|
+
/** Status values emitted by Vitest JSON reports. */
|
|
17
|
+
declare const VitestJsonStatusSchema: z.ZodEnum<{
|
|
18
|
+
passed: "passed";
|
|
19
|
+
failed: "failed";
|
|
20
|
+
skipped: "skipped";
|
|
21
|
+
pending: "pending";
|
|
22
|
+
todo: "todo";
|
|
23
|
+
disabled: "disabled";
|
|
24
|
+
}>;
|
|
25
|
+
/** Status values emitted by Vitest JSON reports. */
|
|
26
|
+
type VitestJsonStatus = z.infer<typeof VitestJsonStatusSchema>;
|
|
27
|
+
/** Source location attached to one Vitest assertion. */
|
|
28
|
+
declare const VitestJsonLocationSchema: z.ZodObject<{
|
|
29
|
+
line: z.ZodNumber;
|
|
30
|
+
column: z.ZodNumber;
|
|
31
|
+
}, z.core.$loose>;
|
|
32
|
+
/** Source location attached to one Vitest assertion. */
|
|
33
|
+
type VitestJsonLocation = z.infer<typeof VitestJsonLocationSchema>;
|
|
34
|
+
/** Assertion record read from Vitest's JSON reporter output. */
|
|
35
|
+
declare const VitestJsonAssertionSchema: z.ZodObject<{
|
|
36
|
+
ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
37
|
+
fullName: z.ZodString;
|
|
38
|
+
status: z.ZodEnum<{
|
|
39
|
+
passed: "passed";
|
|
40
|
+
failed: "failed";
|
|
41
|
+
skipped: "skipped";
|
|
42
|
+
pending: "pending";
|
|
43
|
+
todo: "todo";
|
|
44
|
+
disabled: "disabled";
|
|
45
|
+
}>;
|
|
46
|
+
title: z.ZodString;
|
|
47
|
+
meta: z.ZodOptional<z.ZodUnknown>;
|
|
48
|
+
duration: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
49
|
+
failureMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodString>>>;
|
|
50
|
+
location: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
51
|
+
line: z.ZodNumber;
|
|
52
|
+
column: z.ZodNumber;
|
|
53
|
+
}, z.core.$loose>>>;
|
|
54
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
55
|
+
}, z.core.$loose>;
|
|
56
|
+
/** Assertion record read from Vitest's JSON reporter output. */
|
|
57
|
+
type VitestJsonAssertion = z.infer<typeof VitestJsonAssertionSchema>;
|
|
58
|
+
/** Test-file record read from Vitest's JSON reporter output. */
|
|
59
|
+
declare const VitestJsonFileSchema: z.ZodObject<{
|
|
60
|
+
message: z.ZodString;
|
|
61
|
+
name: z.ZodString;
|
|
62
|
+
status: z.ZodEnum<{
|
|
63
|
+
passed: "passed";
|
|
64
|
+
failed: "failed";
|
|
65
|
+
}>;
|
|
66
|
+
startTime: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
|
|
67
|
+
endTime: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
|
|
68
|
+
assertionResults: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
69
|
+
ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
70
|
+
fullName: z.ZodString;
|
|
71
|
+
status: z.ZodEnum<{
|
|
72
|
+
passed: "passed";
|
|
73
|
+
failed: "failed";
|
|
74
|
+
skipped: "skipped";
|
|
75
|
+
pending: "pending";
|
|
76
|
+
todo: "todo";
|
|
77
|
+
disabled: "disabled";
|
|
78
|
+
}>;
|
|
79
|
+
title: z.ZodString;
|
|
80
|
+
meta: z.ZodOptional<z.ZodUnknown>;
|
|
81
|
+
duration: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
82
|
+
failureMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodString>>>;
|
|
83
|
+
location: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
84
|
+
line: z.ZodNumber;
|
|
85
|
+
column: z.ZodNumber;
|
|
86
|
+
}, z.core.$loose>>>;
|
|
87
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
88
|
+
}, z.core.$loose>>>;
|
|
89
|
+
}, z.core.$loose>;
|
|
90
|
+
/** Test-file record read from Vitest's JSON reporter output. */
|
|
91
|
+
type VitestJsonFile = z.infer<typeof VitestJsonFileSchema>;
|
|
92
|
+
/** Top-level Vitest JSON reporter payload. */
|
|
93
|
+
declare const VitestJsonReportSchema: z.ZodObject<{
|
|
94
|
+
numFailedTests: z.ZodNumber;
|
|
95
|
+
numPassedTests: z.ZodNumber;
|
|
96
|
+
numPendingTests: z.ZodNumber;
|
|
97
|
+
numTodoTests: z.ZodNumber;
|
|
98
|
+
numTotalTests: z.ZodNumber;
|
|
99
|
+
startTime: z.ZodNumber;
|
|
100
|
+
success: z.ZodBoolean;
|
|
101
|
+
testResults: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
102
|
+
message: z.ZodString;
|
|
103
|
+
name: z.ZodString;
|
|
104
|
+
status: z.ZodEnum<{
|
|
105
|
+
passed: "passed";
|
|
106
|
+
failed: "failed";
|
|
107
|
+
}>;
|
|
108
|
+
startTime: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
|
|
109
|
+
endTime: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
|
|
110
|
+
assertionResults: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
111
|
+
ancestorTitles: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
112
|
+
fullName: z.ZodString;
|
|
113
|
+
status: z.ZodEnum<{
|
|
114
|
+
passed: "passed";
|
|
115
|
+
failed: "failed";
|
|
116
|
+
skipped: "skipped";
|
|
117
|
+
pending: "pending";
|
|
118
|
+
todo: "todo";
|
|
119
|
+
disabled: "disabled";
|
|
120
|
+
}>;
|
|
121
|
+
title: z.ZodString;
|
|
122
|
+
meta: z.ZodOptional<z.ZodUnknown>;
|
|
123
|
+
duration: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
124
|
+
failureMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodString>>>;
|
|
125
|
+
location: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
126
|
+
line: z.ZodNumber;
|
|
127
|
+
column: z.ZodNumber;
|
|
128
|
+
}, z.core.$loose>>>;
|
|
129
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
130
|
+
}, z.core.$loose>>>;
|
|
131
|
+
}, z.core.$loose>>>;
|
|
132
|
+
}, z.core.$loose>;
|
|
133
|
+
/** Top-level Vitest JSON reporter payload. */
|
|
134
|
+
type VitestJsonReport = z.infer<typeof VitestJsonReportSchema>;
|
|
135
|
+
/** Parses and validates an unknown value as a Vitest JSON report artifact. */
|
|
136
|
+
declare function parseVitestJsonReport(input: unknown): VitestJsonReport;
|
|
137
|
+
|
|
138
|
+
/** Current schema version for collected report workspaces. */
|
|
139
|
+
declare const REPORT_WORKSPACE_SCHEMA_VERSION = 1;
|
|
140
|
+
/** One collected Vitest JSON report source in a multi-run workspace. */
|
|
141
|
+
declare const ReportRunSchema: z.ZodObject<{
|
|
142
|
+
id: z.ZodString;
|
|
143
|
+
source: z.ZodOptional<z.ZodString>;
|
|
144
|
+
status: z.ZodEnum<{
|
|
145
|
+
passed: "passed";
|
|
146
|
+
failed: "failed";
|
|
147
|
+
}>;
|
|
148
|
+
startedAt: z.ZodOptional<z.ZodNumber>;
|
|
149
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
150
|
+
totals: z.ZodObject<{
|
|
151
|
+
total: z.ZodNumber;
|
|
152
|
+
passed: z.ZodNumber;
|
|
153
|
+
failed: z.ZodNumber;
|
|
154
|
+
skipped: z.ZodNumber;
|
|
155
|
+
evalTotal: z.ZodNumber;
|
|
156
|
+
evalPassed: z.ZodNumber;
|
|
157
|
+
evalFailed: z.ZodNumber;
|
|
158
|
+
}, z.core.$strip>;
|
|
159
|
+
}, z.core.$strict>;
|
|
160
|
+
/** One collected Vitest JSON report source in a multi-run workspace. */
|
|
161
|
+
type ReportRun = z.infer<typeof ReportRunSchema>;
|
|
162
|
+
/** One eval or harness-backed test case collected from Vitest JSON. */
|
|
163
|
+
declare const ReportCaseSchema: z.ZodObject<{
|
|
164
|
+
id: z.ZodString;
|
|
165
|
+
runId: z.ZodString;
|
|
166
|
+
source: z.ZodOptional<z.ZodString>;
|
|
167
|
+
file: z.ZodString;
|
|
168
|
+
displayFile: z.ZodString;
|
|
169
|
+
title: z.ZodString;
|
|
170
|
+
fullName: z.ZodString;
|
|
171
|
+
ancestorTitles: z.ZodArray<z.ZodString>;
|
|
172
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
173
|
+
displayName: z.ZodString;
|
|
174
|
+
status: z.ZodEnum<{
|
|
175
|
+
passed: "passed";
|
|
176
|
+
failed: "failed";
|
|
177
|
+
skipped: "skipped";
|
|
178
|
+
pending: "pending";
|
|
179
|
+
todo: "todo";
|
|
180
|
+
disabled: "disabled";
|
|
181
|
+
}>;
|
|
182
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
183
|
+
location: z.ZodOptional<z.ZodObject<{
|
|
184
|
+
line: z.ZodNumber;
|
|
185
|
+
column: z.ZodNumber;
|
|
186
|
+
}, z.core.$loose>>;
|
|
187
|
+
failureMessages: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
188
|
+
eval: z.ZodOptional<z.ZodObject<{
|
|
189
|
+
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
190
|
+
name: z.ZodOptional<z.ZodString>;
|
|
191
|
+
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
192
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
193
|
+
}, z.core.$strict>>>;
|
|
194
|
+
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
195
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
196
|
+
thresholdFailed: z.ZodOptional<z.ZodBoolean>;
|
|
197
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
198
|
+
id: z.ZodOptional<z.ZodString>;
|
|
199
|
+
name: z.ZodString;
|
|
200
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
201
|
+
result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
202
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
203
|
+
message: z.ZodString;
|
|
204
|
+
type: z.ZodOptional<z.ZodString>;
|
|
205
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
206
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
207
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
208
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
209
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
210
|
+
}, z.core.$strict>>>;
|
|
211
|
+
}, z.core.$strict>>;
|
|
212
|
+
harness: z.ZodOptional<z.ZodObject<{
|
|
213
|
+
name: z.ZodOptional<z.ZodString>;
|
|
214
|
+
run: z.ZodOptional<z.ZodObject<{
|
|
215
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
216
|
+
session: z.ZodObject<{
|
|
217
|
+
messages: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
218
|
+
role: z.ZodEnum<{
|
|
219
|
+
system: "system";
|
|
220
|
+
user: "user";
|
|
221
|
+
assistant: "assistant";
|
|
222
|
+
tool: "tool";
|
|
223
|
+
}>;
|
|
224
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
225
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
226
|
+
id: z.ZodOptional<z.ZodString>;
|
|
227
|
+
name: z.ZodString;
|
|
228
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
229
|
+
result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
230
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
231
|
+
message: z.ZodString;
|
|
232
|
+
type: z.ZodOptional<z.ZodString>;
|
|
233
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
234
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
235
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
236
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
237
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
238
|
+
}, z.core.$strict>>>;
|
|
239
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
240
|
+
}, z.core.$strict>>>;
|
|
241
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
242
|
+
model: z.ZodOptional<z.ZodString>;
|
|
243
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
244
|
+
}, z.core.$strict>;
|
|
245
|
+
usage: z.ZodObject<{
|
|
246
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
247
|
+
model: z.ZodOptional<z.ZodString>;
|
|
248
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
249
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
250
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
251
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
252
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
253
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
254
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
255
|
+
}, z.core.$strict>;
|
|
256
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
257
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
258
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
259
|
+
}, z.core.$strict>>;
|
|
260
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
261
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
262
|
+
id: z.ZodOptional<z.ZodString>;
|
|
263
|
+
name: z.ZodOptional<z.ZodString>;
|
|
264
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
265
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
266
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
267
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
268
|
+
spans: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
269
|
+
id: z.ZodOptional<z.ZodString>;
|
|
270
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
271
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
272
|
+
name: z.ZodString;
|
|
273
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
274
|
+
custom: "custom";
|
|
275
|
+
model: "model";
|
|
276
|
+
tool: "tool";
|
|
277
|
+
run: "run";
|
|
278
|
+
agent: "agent";
|
|
279
|
+
guardrail: "guardrail";
|
|
280
|
+
handoff: "handoff";
|
|
281
|
+
}>>;
|
|
282
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
283
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
284
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
285
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
286
|
+
error: "error";
|
|
287
|
+
ok: "ok";
|
|
288
|
+
}>>;
|
|
289
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
290
|
+
message: z.ZodString;
|
|
291
|
+
type: z.ZodOptional<z.ZodString>;
|
|
292
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
293
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
294
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
295
|
+
name: z.ZodString;
|
|
296
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
297
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
298
|
+
}, z.core.$strict>>>;
|
|
299
|
+
}, z.core.$strict>>>;
|
|
300
|
+
}, z.core.$strict>>>;
|
|
301
|
+
errors: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
302
|
+
}, z.core.$strict>>;
|
|
303
|
+
}, z.core.$strict>>;
|
|
304
|
+
}, z.core.$strict>;
|
|
305
|
+
/** One eval or harness-backed test case collected from Vitest JSON. */
|
|
306
|
+
type ReportCase = z.infer<typeof ReportCaseSchema>;
|
|
307
|
+
/** Full multi-run report workspace consumed by rich report UIs. */
|
|
308
|
+
declare const ReportWorkspaceSchema: z.ZodObject<{
|
|
309
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
310
|
+
runs: z.ZodArray<z.ZodObject<{
|
|
311
|
+
id: z.ZodString;
|
|
312
|
+
source: z.ZodOptional<z.ZodString>;
|
|
313
|
+
status: z.ZodEnum<{
|
|
314
|
+
passed: "passed";
|
|
315
|
+
failed: "failed";
|
|
316
|
+
}>;
|
|
317
|
+
startedAt: z.ZodOptional<z.ZodNumber>;
|
|
318
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
319
|
+
totals: z.ZodObject<{
|
|
320
|
+
total: z.ZodNumber;
|
|
321
|
+
passed: z.ZodNumber;
|
|
322
|
+
failed: z.ZodNumber;
|
|
323
|
+
skipped: z.ZodNumber;
|
|
324
|
+
evalTotal: z.ZodNumber;
|
|
325
|
+
evalPassed: z.ZodNumber;
|
|
326
|
+
evalFailed: z.ZodNumber;
|
|
327
|
+
}, z.core.$strip>;
|
|
328
|
+
}, z.core.$strict>>;
|
|
329
|
+
cases: z.ZodArray<z.ZodObject<{
|
|
330
|
+
id: z.ZodString;
|
|
331
|
+
runId: z.ZodString;
|
|
332
|
+
source: z.ZodOptional<z.ZodString>;
|
|
333
|
+
file: z.ZodString;
|
|
334
|
+
displayFile: z.ZodString;
|
|
335
|
+
title: z.ZodString;
|
|
336
|
+
fullName: z.ZodString;
|
|
337
|
+
ancestorTitles: z.ZodArray<z.ZodString>;
|
|
338
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
339
|
+
displayName: z.ZodString;
|
|
340
|
+
status: z.ZodEnum<{
|
|
341
|
+
passed: "passed";
|
|
342
|
+
failed: "failed";
|
|
343
|
+
skipped: "skipped";
|
|
344
|
+
pending: "pending";
|
|
345
|
+
todo: "todo";
|
|
346
|
+
disabled: "disabled";
|
|
347
|
+
}>;
|
|
348
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
349
|
+
location: z.ZodOptional<z.ZodObject<{
|
|
350
|
+
line: z.ZodNumber;
|
|
351
|
+
column: z.ZodNumber;
|
|
352
|
+
}, z.core.$loose>>;
|
|
353
|
+
failureMessages: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
354
|
+
eval: z.ZodOptional<z.ZodObject<{
|
|
355
|
+
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
356
|
+
name: z.ZodOptional<z.ZodString>;
|
|
357
|
+
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
358
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
359
|
+
}, z.core.$strict>>>;
|
|
360
|
+
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
361
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
362
|
+
thresholdFailed: z.ZodOptional<z.ZodBoolean>;
|
|
363
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
364
|
+
id: z.ZodOptional<z.ZodString>;
|
|
365
|
+
name: z.ZodString;
|
|
366
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
367
|
+
result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
368
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
369
|
+
message: z.ZodString;
|
|
370
|
+
type: z.ZodOptional<z.ZodString>;
|
|
371
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
372
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
373
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
374
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
375
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
376
|
+
}, z.core.$strict>>>;
|
|
377
|
+
}, z.core.$strict>>;
|
|
378
|
+
harness: z.ZodOptional<z.ZodObject<{
|
|
379
|
+
name: z.ZodOptional<z.ZodString>;
|
|
380
|
+
run: z.ZodOptional<z.ZodObject<{
|
|
381
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
382
|
+
session: z.ZodObject<{
|
|
383
|
+
messages: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
384
|
+
role: z.ZodEnum<{
|
|
385
|
+
system: "system";
|
|
386
|
+
user: "user";
|
|
387
|
+
assistant: "assistant";
|
|
388
|
+
tool: "tool";
|
|
389
|
+
}>;
|
|
390
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
391
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
392
|
+
id: z.ZodOptional<z.ZodString>;
|
|
393
|
+
name: z.ZodString;
|
|
394
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
395
|
+
result: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
396
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
397
|
+
message: z.ZodString;
|
|
398
|
+
type: z.ZodOptional<z.ZodString>;
|
|
399
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
400
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
401
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
402
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
403
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
404
|
+
}, z.core.$strict>>>;
|
|
405
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
406
|
+
}, z.core.$strict>>>;
|
|
407
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
408
|
+
model: z.ZodOptional<z.ZodString>;
|
|
409
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
410
|
+
}, z.core.$strict>;
|
|
411
|
+
usage: z.ZodObject<{
|
|
412
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
413
|
+
model: z.ZodOptional<z.ZodString>;
|
|
414
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
415
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
416
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
417
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
418
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
419
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
420
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
421
|
+
}, z.core.$strict>;
|
|
422
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
423
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
424
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
425
|
+
}, z.core.$strict>>;
|
|
426
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
427
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
428
|
+
id: z.ZodOptional<z.ZodString>;
|
|
429
|
+
name: z.ZodOptional<z.ZodString>;
|
|
430
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
431
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
432
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
433
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
434
|
+
spans: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
435
|
+
id: z.ZodOptional<z.ZodString>;
|
|
436
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
437
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
438
|
+
name: z.ZodString;
|
|
439
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
440
|
+
custom: "custom";
|
|
441
|
+
model: "model";
|
|
442
|
+
tool: "tool";
|
|
443
|
+
run: "run";
|
|
444
|
+
agent: "agent";
|
|
445
|
+
guardrail: "guardrail";
|
|
446
|
+
handoff: "handoff";
|
|
447
|
+
}>>;
|
|
448
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
449
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
450
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
451
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
452
|
+
error: "error";
|
|
453
|
+
ok: "ok";
|
|
454
|
+
}>>;
|
|
455
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
456
|
+
message: z.ZodString;
|
|
457
|
+
type: z.ZodOptional<z.ZodString>;
|
|
458
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
459
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
460
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
461
|
+
name: z.ZodString;
|
|
462
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
463
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
464
|
+
}, z.core.$strict>>>;
|
|
465
|
+
}, z.core.$strict>>>;
|
|
466
|
+
}, z.core.$strict>>>;
|
|
467
|
+
errors: z.ZodDefault<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
468
|
+
}, z.core.$strict>>;
|
|
469
|
+
}, z.core.$strict>>;
|
|
470
|
+
}, z.core.$strict>>;
|
|
471
|
+
}, z.core.$strict>;
|
|
472
|
+
/** Full multi-run report workspace consumed by rich report UIs. */
|
|
473
|
+
type ReportWorkspace = z.infer<typeof ReportWorkspaceSchema>;
|
|
474
|
+
/** Input accepted when collecting one Vitest JSON report into a workspace. */
|
|
475
|
+
type ReportWorkspaceInput = VitestJsonReport | {
|
|
476
|
+
report: VitestJsonReport;
|
|
477
|
+
source?: string;
|
|
478
|
+
};
|
|
479
|
+
/** Options for collecting one or more Vitest JSON reports. */
|
|
480
|
+
type CollectReportWorkspaceOptions = {
|
|
481
|
+
/** Workspace prefix used to render source files as relative paths. */
|
|
482
|
+
workspace?: string;
|
|
483
|
+
};
|
|
484
|
+
/** Parses and validates an unknown value as a collected report workspace. */
|
|
485
|
+
declare function parseReportWorkspace(input: unknown): ReportWorkspace;
|
|
486
|
+
/** Collects eval and harness metadata from one or more Vitest JSON reports. */
|
|
487
|
+
declare function collectReportWorkspace(input: ReportWorkspaceInput | ReportWorkspaceInput[], options?: CollectReportWorkspaceOptions): ReportWorkspace;
|
|
488
|
+
|
|
489
|
+
export { type CollectReportWorkspaceOptions as C, type JsonValue as J, REPORT_WORKSPACE_SCHEMA_VERSION as R, VitestJsonAssertionSchema as V, JsonObjectSchema as a, JsonPrimitiveSchema as b, JsonValueSchema as c, type JsonPrimitive as d, VitestJsonFileSchema as e, VitestJsonLocationSchema as f, VitestJsonReportSchema as g, VitestJsonStatusSchema as h, type VitestJsonAssertion as i, type VitestJsonFile as j, type VitestJsonLocation as k, type VitestJsonReport as l, type VitestJsonStatus as m, collectReportWorkspace as n, parseReportWorkspace as o, parseVitestJsonReport as p, ReportCaseSchema as q, ReportRunSchema as r, ReportWorkspaceSchema as s, type ReportCase as t, type ReportRun as u, type ReportWorkspace as v, type ReportWorkspaceInput as w };
|
package/package.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@vitest-evals/core",
|
|
3
|
+
"version": "0.12.0",
|
|
4
|
+
"description": "Shared primitives, schemas, and report collection helpers for vitest-evals.",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/getsentry/vitest-evals.git",
|
|
8
|
+
"directory": "packages/core"
|
|
9
|
+
},
|
|
10
|
+
"author": "David Cramer",
|
|
11
|
+
"license": "Apache-2.0",
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/getsentry/vitest-evals/issues"
|
|
14
|
+
},
|
|
15
|
+
"homepage": "https://github.com/getsentry/vitest-evals/tree/main/packages/core#readme",
|
|
16
|
+
"sideEffects": false,
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"main": "./dist/index.js",
|
|
19
|
+
"module": "./dist/index.mjs",
|
|
20
|
+
"files": [
|
|
21
|
+
"dist"
|
|
22
|
+
],
|
|
23
|
+
"publishConfig": {
|
|
24
|
+
"access": "public"
|
|
25
|
+
},
|
|
26
|
+
"exports": {
|
|
27
|
+
".": {
|
|
28
|
+
"source": "./src/index.ts",
|
|
29
|
+
"types": "./dist/index.d.ts",
|
|
30
|
+
"require": "./dist/index.js",
|
|
31
|
+
"import": "./dist/index.mjs"
|
|
32
|
+
},
|
|
33
|
+
"./node": {
|
|
34
|
+
"source": "./src/node.ts",
|
|
35
|
+
"types": "./dist/node.d.ts",
|
|
36
|
+
"require": "./dist/node.js",
|
|
37
|
+
"import": "./dist/node.mjs"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"zod": ">=3 <5"
|
|
42
|
+
},
|
|
43
|
+
"scripts": {
|
|
44
|
+
"build": "tsup --config ./tsup.config.ts"
|
|
45
|
+
}
|
|
46
|
+
}
|