@vitest-evals/core 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +36 -0
- package/dist/index.d.mts +895 -0
- package/dist/index.d.ts +895 -0
- package/dist/index.js +557 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +489 -0
- package/dist/index.mjs.map +1 -0
- package/dist/node.d.mts +33 -0
- package/dist/node.d.ts +33 -0
- package/dist/node.js +628 -0
- package/dist/node.js.map +1 -0
- package/dist/node.mjs +598 -0
- package/dist/node.mjs.map +1 -0
- package/dist/workspace-BNgjyW7W.d.mts +489 -0
- package/dist/workspace-BNgjyW7W.d.ts +489 -0
- package/package.json +46 -0
package/dist/node.js
ADDED
|
@@ -0,0 +1,628 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/node.ts
|
|
21
|
+
var node_exports = {};
|
|
22
|
+
__export(node_exports, {
|
|
23
|
+
readReportWorkspace: () => readReportWorkspace,
|
|
24
|
+
readVitestJsonReportFile: () => readVitestJsonReportFile,
|
|
25
|
+
resolveReportFiles: () => resolveReportFiles,
|
|
26
|
+
resolveResultFiles: () => resolveResultFiles,
|
|
27
|
+
splitReportInput: () => splitReportInput,
|
|
28
|
+
splitResultsInput: () => splitResultsInput
|
|
29
|
+
});
|
|
30
|
+
module.exports = __toCommonJS(node_exports);
|
|
31
|
+
var import_promises = require("fs/promises");
|
|
32
|
+
var import_node_path = require("path");
|
|
33
|
+
|
|
34
|
+
// src/report/workspace.ts
|
|
35
|
+
var import_zod6 = require("zod");
|
|
36
|
+
|
|
37
|
+
// src/report/metadata.ts
|
|
38
|
+
var import_zod4 = require("zod");
|
|
39
|
+
|
|
40
|
+
// src/harness/index.ts
|
|
41
|
+
var import_zod3 = require("zod");
|
|
42
|
+
|
|
43
|
+
// src/json.ts
|
|
44
|
+
var import_zod = require("zod");
|
|
45
|
+
var JsonPrimitiveSchema = import_zod.z.union([
|
|
46
|
+
import_zod.z.string(),
|
|
47
|
+
import_zod.z.number().finite(),
|
|
48
|
+
import_zod.z.boolean(),
|
|
49
|
+
import_zod.z.null()
|
|
50
|
+
]);
|
|
51
|
+
var JsonValueSchema = import_zod.z.lazy(
|
|
52
|
+
() => import_zod.z.union([
|
|
53
|
+
JsonPrimitiveSchema,
|
|
54
|
+
import_zod.z.array(JsonValueSchema),
|
|
55
|
+
import_zod.z.record(import_zod.z.string(), JsonValueSchema)
|
|
56
|
+
])
|
|
57
|
+
);
|
|
58
|
+
var JsonObjectSchema = import_zod.z.record(import_zod.z.string(), JsonValueSchema);
|
|
59
|
+
|
|
60
|
+
// src/schema-utils.ts
|
|
61
|
+
var import_zod2 = require("zod");
|
|
62
|
+
var FiniteNumberSchema = import_zod2.z.number().finite();
|
|
63
|
+
var OptionalFiniteNumberSchema = import_zod2.z.preprocess(
|
|
64
|
+
(value) => value === null || typeof value === "number" && !Number.isFinite(value) ? void 0 : value,
|
|
65
|
+
FiniteNumberSchema.optional()
|
|
66
|
+
);
|
|
67
|
+
var NullableFiniteNumberSchema = import_zod2.z.preprocess(
|
|
68
|
+
(value) => typeof value === "number" && !Number.isFinite(value) ? null : value,
|
|
69
|
+
FiniteNumberSchema.nullable().optional()
|
|
70
|
+
);
|
|
71
|
+
function parseWithSchema(schema, input, label) {
|
|
72
|
+
const parsed = schema.safeParse(input);
|
|
73
|
+
if (parsed.success) {
|
|
74
|
+
return parsed.data;
|
|
75
|
+
}
|
|
76
|
+
const reason = parsed.error.issues.slice(0, 3).map((issue) => {
|
|
77
|
+
const path = issue.path.length > 0 ? issue.path.join(".") : "<root>";
|
|
78
|
+
return `${path}: ${issue.message}`;
|
|
79
|
+
}).join("; ");
|
|
80
|
+
throw new Error(`Invalid ${label}: ${reason}`);
|
|
81
|
+
}
|
|
82
|
+
function isJsonObject(value) {
|
|
83
|
+
return isRecord(value);
|
|
84
|
+
}
|
|
85
|
+
function isRecord(value) {
|
|
86
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// src/harness/index.ts
|
|
90
|
+
var UsageSummarySchema = import_zod3.z.object({
|
|
91
|
+
provider: import_zod3.z.string().optional(),
|
|
92
|
+
model: import_zod3.z.string().optional(),
|
|
93
|
+
inputTokens: FiniteNumberSchema.optional(),
|
|
94
|
+
outputTokens: FiniteNumberSchema.optional(),
|
|
95
|
+
reasoningTokens: FiniteNumberSchema.optional(),
|
|
96
|
+
totalTokens: FiniteNumberSchema.optional(),
|
|
97
|
+
toolCalls: FiniteNumberSchema.optional(),
|
|
98
|
+
retries: FiniteNumberSchema.optional(),
|
|
99
|
+
metadata: JsonObjectSchema.optional()
|
|
100
|
+
}).strict();
|
|
101
|
+
var TimingSummarySchema = import_zod3.z.object({
|
|
102
|
+
totalMs: FiniteNumberSchema.optional(),
|
|
103
|
+
metadata: JsonObjectSchema.optional()
|
|
104
|
+
}).strict();
|
|
105
|
+
var NormalizedErrorSchema = import_zod3.z.object({
|
|
106
|
+
message: import_zod3.z.string(),
|
|
107
|
+
type: import_zod3.z.string().optional()
|
|
108
|
+
}).catchall(JsonValueSchema);
|
|
109
|
+
var ToolCallRecordSchema = import_zod3.z.object({
|
|
110
|
+
id: import_zod3.z.string().optional(),
|
|
111
|
+
name: import_zod3.z.string(),
|
|
112
|
+
arguments: JsonObjectSchema.optional(),
|
|
113
|
+
result: JsonValueSchema.optional(),
|
|
114
|
+
error: NormalizedErrorSchema.optional(),
|
|
115
|
+
startedAt: import_zod3.z.string().optional(),
|
|
116
|
+
finishedAt: import_zod3.z.string().optional(),
|
|
117
|
+
durationMs: FiniteNumberSchema.optional(),
|
|
118
|
+
metadata: JsonObjectSchema.optional()
|
|
119
|
+
}).strict();
|
|
120
|
+
var NormalizedMessageSchema = import_zod3.z.object({
|
|
121
|
+
role: import_zod3.z.enum(["system", "user", "assistant", "tool"]),
|
|
122
|
+
content: JsonValueSchema.optional(),
|
|
123
|
+
toolCalls: import_zod3.z.array(ToolCallRecordSchema).optional(),
|
|
124
|
+
metadata: JsonObjectSchema.optional()
|
|
125
|
+
}).strict();
|
|
126
|
+
var NormalizedSessionSchema = import_zod3.z.object({
|
|
127
|
+
messages: import_zod3.z.array(NormalizedMessageSchema).default([]),
|
|
128
|
+
provider: import_zod3.z.string().optional(),
|
|
129
|
+
model: import_zod3.z.string().optional(),
|
|
130
|
+
metadata: JsonObjectSchema.optional()
|
|
131
|
+
}).strict();
|
|
132
|
+
var NormalizedSpanAttributesSchema = JsonObjectSchema;
|
|
133
|
+
var NormalizedSpanEventSchema = import_zod3.z.object({
|
|
134
|
+
name: import_zod3.z.string(),
|
|
135
|
+
timestamp: import_zod3.z.string().optional(),
|
|
136
|
+
attributes: NormalizedSpanAttributesSchema.optional()
|
|
137
|
+
}).strict();
|
|
138
|
+
var NormalizedSpanSchema = import_zod3.z.object({
|
|
139
|
+
id: import_zod3.z.string().optional(),
|
|
140
|
+
traceId: import_zod3.z.string().optional(),
|
|
141
|
+
parentId: import_zod3.z.string().optional(),
|
|
142
|
+
name: import_zod3.z.string(),
|
|
143
|
+
kind: import_zod3.z.enum(["run", "agent", "model", "tool", "guardrail", "handoff", "custom"]).optional(),
|
|
144
|
+
startedAt: import_zod3.z.string().optional(),
|
|
145
|
+
finishedAt: import_zod3.z.string().optional(),
|
|
146
|
+
durationMs: FiniteNumberSchema.optional(),
|
|
147
|
+
status: import_zod3.z.enum(["ok", "error"]).optional(),
|
|
148
|
+
error: NormalizedErrorSchema.optional(),
|
|
149
|
+
attributes: NormalizedSpanAttributesSchema.optional(),
|
|
150
|
+
events: import_zod3.z.array(NormalizedSpanEventSchema).optional()
|
|
151
|
+
}).strict();
|
|
152
|
+
var NormalizedTraceSchema = import_zod3.z.object({
|
|
153
|
+
id: import_zod3.z.string().optional(),
|
|
154
|
+
name: import_zod3.z.string().optional(),
|
|
155
|
+
startedAt: import_zod3.z.string().optional(),
|
|
156
|
+
finishedAt: import_zod3.z.string().optional(),
|
|
157
|
+
durationMs: FiniteNumberSchema.optional(),
|
|
158
|
+
metadata: JsonObjectSchema.optional(),
|
|
159
|
+
spans: import_zod3.z.array(NormalizedSpanSchema).default([])
|
|
160
|
+
}).strict();
|
|
161
|
+
var HarnessRunSchema = import_zod3.z.object({
|
|
162
|
+
output: JsonValueSchema.optional(),
|
|
163
|
+
session: NormalizedSessionSchema,
|
|
164
|
+
usage: UsageSummarySchema,
|
|
165
|
+
timings: TimingSummarySchema.optional(),
|
|
166
|
+
artifacts: JsonObjectSchema.optional(),
|
|
167
|
+
traces: import_zod3.z.array(NormalizedTraceSchema).optional(),
|
|
168
|
+
errors: import_zod3.z.array(JsonObjectSchema).default([])
|
|
169
|
+
}).strict();
|
|
170
|
+
|
|
171
|
+
// src/report/metadata.ts
|
|
172
|
+
var HarnessMetaSchema = import_zod4.z.object({
|
|
173
|
+
name: import_zod4.z.string().optional(),
|
|
174
|
+
run: HarnessRunSchema.optional()
|
|
175
|
+
}).strict();
|
|
176
|
+
var EvalScoreSchema = import_zod4.z.object({
|
|
177
|
+
name: import_zod4.z.string().optional(),
|
|
178
|
+
score: NullableFiniteNumberSchema,
|
|
179
|
+
metadata: JsonObjectSchema.optional()
|
|
180
|
+
}).strict();
|
|
181
|
+
var EvalMetaSchema = import_zod4.z.object({
|
|
182
|
+
scores: import_zod4.z.array(EvalScoreSchema).optional(),
|
|
183
|
+
avgScore: NullableFiniteNumberSchema,
|
|
184
|
+
output: JsonValueSchema.optional(),
|
|
185
|
+
thresholdFailed: import_zod4.z.boolean().optional(),
|
|
186
|
+
toolCalls: import_zod4.z.array(ToolCallRecordSchema).optional()
|
|
187
|
+
}).strict();
|
|
188
|
+
var EvalTaskMetaSchema = import_zod4.z.object({
|
|
189
|
+
eval: EvalMetaSchema.optional(),
|
|
190
|
+
harness: HarnessMetaSchema.optional()
|
|
191
|
+
}).strict();
|
|
192
|
+
var LenientToolCallRecordSchema = ToolCallRecordSchema.strip();
|
|
193
|
+
var LenientMessageSchema = NormalizedMessageSchema.extend({
|
|
194
|
+
toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
|
|
195
|
+
}).strip();
|
|
196
|
+
var LenientSessionSchema = NormalizedSessionSchema.extend({
|
|
197
|
+
messages: import_zod4.z.array(LenientMessageSchema).default([]).catch([])
|
|
198
|
+
}).strip();
|
|
199
|
+
var LenientSpanEventSchema = NormalizedSpanEventSchema.strip();
|
|
200
|
+
var LenientSpanSchema = NormalizedSpanSchema.extend({
|
|
201
|
+
events: import_zod4.z.array(LenientSpanEventSchema).optional().catch(void 0)
|
|
202
|
+
}).strip();
|
|
203
|
+
var LenientTraceSchema = NormalizedTraceSchema.extend({
|
|
204
|
+
spans: import_zod4.z.array(LenientSpanSchema).default([]).catch([])
|
|
205
|
+
}).strip();
|
|
206
|
+
var LenientHarnessRunSchema = HarnessRunSchema.extend({
|
|
207
|
+
session: LenientSessionSchema,
|
|
208
|
+
usage: UsageSummarySchema.strip().default({}),
|
|
209
|
+
timings: TimingSummarySchema.strip().optional(),
|
|
210
|
+
traces: import_zod4.z.array(LenientTraceSchema).optional().catch(void 0)
|
|
211
|
+
}).strip();
|
|
212
|
+
var LenientHarnessMetaSchema = HarnessMetaSchema.extend({
|
|
213
|
+
run: LenientHarnessRunSchema.optional().catch(void 0)
|
|
214
|
+
}).strip();
|
|
215
|
+
var LenientEvalScoreSchema = EvalScoreSchema.strip();
|
|
216
|
+
var LenientEvalMetaSchema = EvalMetaSchema.extend({
|
|
217
|
+
scores: import_zod4.z.array(LenientEvalScoreSchema).optional().catch(void 0),
|
|
218
|
+
toolCalls: import_zod4.z.array(LenientToolCallRecordSchema).optional().catch(void 0)
|
|
219
|
+
}).strip();
|
|
220
|
+
function readEvalTaskMeta(input) {
|
|
221
|
+
if (!isJsonObject(input)) {
|
|
222
|
+
return void 0;
|
|
223
|
+
}
|
|
224
|
+
const evalResult = LenientEvalMetaSchema.safeParse(input.eval);
|
|
225
|
+
const harnessResult = LenientHarnessMetaSchema.safeParse(input.harness);
|
|
226
|
+
const meta = {
|
|
227
|
+
...evalResult.success && input.eval !== void 0 ? { eval: evalResult.data } : {},
|
|
228
|
+
...harnessResult.success && input.harness !== void 0 ? { harness: harnessResult.data } : {}
|
|
229
|
+
};
|
|
230
|
+
return meta.eval || meta.harness ? meta : void 0;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// src/report/vitest-json.ts
|
|
234
|
+
var import_zod5 = require("zod");
|
|
235
|
+
var VitestJsonStatusSchema = import_zod5.z.enum([
|
|
236
|
+
"passed",
|
|
237
|
+
"failed",
|
|
238
|
+
"skipped",
|
|
239
|
+
"pending",
|
|
240
|
+
"todo",
|
|
241
|
+
"disabled"
|
|
242
|
+
]);
|
|
243
|
+
var VitestJsonLocationSchema = import_zod5.z.object({
|
|
244
|
+
line: FiniteNumberSchema,
|
|
245
|
+
column: FiniteNumberSchema
|
|
246
|
+
}).passthrough();
|
|
247
|
+
var VitestJsonAssertionSchema = import_zod5.z.object({
|
|
248
|
+
ancestorTitles: import_zod5.z.array(import_zod5.z.string()).default([]),
|
|
249
|
+
fullName: import_zod5.z.string(),
|
|
250
|
+
status: VitestJsonStatusSchema,
|
|
251
|
+
title: import_zod5.z.string(),
|
|
252
|
+
meta: import_zod5.z.unknown().optional(),
|
|
253
|
+
duration: FiniteNumberSchema.nullable().optional(),
|
|
254
|
+
failureMessages: import_zod5.z.array(import_zod5.z.string()).nullable().optional(),
|
|
255
|
+
location: VitestJsonLocationSchema.nullable().optional(),
|
|
256
|
+
tags: import_zod5.z.array(import_zod5.z.string()).optional()
|
|
257
|
+
}).passthrough();
|
|
258
|
+
var VitestJsonFileSchema = import_zod5.z.object({
|
|
259
|
+
message: import_zod5.z.string(),
|
|
260
|
+
name: import_zod5.z.string(),
|
|
261
|
+
status: import_zod5.z.enum(["failed", "passed"]),
|
|
262
|
+
startTime: OptionalFiniteNumberSchema,
|
|
263
|
+
endTime: OptionalFiniteNumberSchema,
|
|
264
|
+
assertionResults: import_zod5.z.array(VitestJsonAssertionSchema).default([])
|
|
265
|
+
}).passthrough();
|
|
266
|
+
var VitestJsonReportSchema = import_zod5.z.object({
|
|
267
|
+
numFailedTests: FiniteNumberSchema,
|
|
268
|
+
numPassedTests: FiniteNumberSchema,
|
|
269
|
+
numPendingTests: FiniteNumberSchema,
|
|
270
|
+
numTodoTests: FiniteNumberSchema,
|
|
271
|
+
numTotalTests: FiniteNumberSchema,
|
|
272
|
+
startTime: FiniteNumberSchema,
|
|
273
|
+
success: import_zod5.z.boolean(),
|
|
274
|
+
testResults: import_zod5.z.array(VitestJsonFileSchema).default([])
|
|
275
|
+
}).passthrough();
|
|
276
|
+
function parseVitestJsonReport(input) {
|
|
277
|
+
return parseWithSchema(VitestJsonReportSchema, input, "Vitest JSON report");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// src/report/workspace.ts
|
|
281
|
+
var REPORT_WORKSPACE_SCHEMA_VERSION = 1;
|
|
282
|
+
var ReportRunSchema = import_zod6.z.object({
|
|
283
|
+
id: import_zod6.z.string(),
|
|
284
|
+
source: import_zod6.z.string().optional(),
|
|
285
|
+
status: import_zod6.z.enum(["passed", "failed"]),
|
|
286
|
+
startedAt: FiniteNumberSchema.optional(),
|
|
287
|
+
durationMs: FiniteNumberSchema.optional(),
|
|
288
|
+
totals: import_zod6.z.object({
|
|
289
|
+
total: FiniteNumberSchema,
|
|
290
|
+
passed: FiniteNumberSchema,
|
|
291
|
+
failed: FiniteNumberSchema,
|
|
292
|
+
skipped: FiniteNumberSchema,
|
|
293
|
+
evalTotal: FiniteNumberSchema,
|
|
294
|
+
evalPassed: FiniteNumberSchema,
|
|
295
|
+
evalFailed: FiniteNumberSchema
|
|
296
|
+
})
|
|
297
|
+
}).strict();
|
|
298
|
+
var ReportCaseSchema = import_zod6.z.object({
|
|
299
|
+
id: import_zod6.z.string(),
|
|
300
|
+
runId: import_zod6.z.string(),
|
|
301
|
+
source: import_zod6.z.string().optional(),
|
|
302
|
+
file: import_zod6.z.string(),
|
|
303
|
+
displayFile: import_zod6.z.string(),
|
|
304
|
+
title: import_zod6.z.string(),
|
|
305
|
+
fullName: import_zod6.z.string(),
|
|
306
|
+
ancestorTitles: import_zod6.z.array(import_zod6.z.string()),
|
|
307
|
+
tags: import_zod6.z.array(import_zod6.z.string()).optional(),
|
|
308
|
+
displayName: import_zod6.z.string(),
|
|
309
|
+
status: VitestJsonStatusSchema,
|
|
310
|
+
durationMs: FiniteNumberSchema.optional(),
|
|
311
|
+
location: VitestJsonLocationSchema.optional(),
|
|
312
|
+
failureMessages: import_zod6.z.array(import_zod6.z.string()).default([]),
|
|
313
|
+
eval: EvalMetaSchema.optional(),
|
|
314
|
+
harness: HarnessMetaSchema.optional()
|
|
315
|
+
}).strict();
|
|
316
|
+
var ReportWorkspaceSchema = import_zod6.z.object({
|
|
317
|
+
schemaVersion: import_zod6.z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
|
|
318
|
+
runs: import_zod6.z.array(ReportRunSchema),
|
|
319
|
+
cases: import_zod6.z.array(ReportCaseSchema)
|
|
320
|
+
}).strict();
|
|
321
|
+
function collectReportWorkspace(input, options = {}) {
|
|
322
|
+
const entries = Array.isArray(input) ? input : [input];
|
|
323
|
+
const runs = [];
|
|
324
|
+
const cases = [];
|
|
325
|
+
entries.forEach((entry, index) => {
|
|
326
|
+
const { report, source } = normalizeWorkspaceInput(entry);
|
|
327
|
+
const runId = source ?? `run-${index + 1}`;
|
|
328
|
+
const runCases = [];
|
|
329
|
+
for (const file of report.testResults) {
|
|
330
|
+
for (const assertion of file.assertionResults) {
|
|
331
|
+
const meta = readEvalTaskMeta(assertion.meta);
|
|
332
|
+
if (!meta) {
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
runCases.push({
|
|
336
|
+
id: createCaseId(runId, file.name, assertion),
|
|
337
|
+
runId,
|
|
338
|
+
...source ? { source } : {},
|
|
339
|
+
file: file.name,
|
|
340
|
+
displayFile: normalizeReportPath(file.name, options.workspace),
|
|
341
|
+
title: assertion.title,
|
|
342
|
+
fullName: assertion.fullName,
|
|
343
|
+
ancestorTitles: assertion.ancestorTitles,
|
|
344
|
+
...assertion.tags ? { tags: assertion.tags } : {},
|
|
345
|
+
displayName: formatDisplayName(assertion),
|
|
346
|
+
status: assertion.status,
|
|
347
|
+
...typeof assertion.duration === "number" ? { durationMs: assertion.duration } : {},
|
|
348
|
+
...assertion.location ? { location: assertion.location } : {},
|
|
349
|
+
failureMessages: assertion.failureMessages ?? [],
|
|
350
|
+
...meta.eval ? { eval: meta.eval } : {},
|
|
351
|
+
...meta.harness ? { harness: meta.harness } : {}
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
runs.push({
|
|
356
|
+
id: runId,
|
|
357
|
+
...source ? { source } : {},
|
|
358
|
+
status: report.success && runCases.every((testCase) => testCase.status !== "failed") ? "passed" : "failed",
|
|
359
|
+
startedAt: report.startTime,
|
|
360
|
+
durationMs: resolveRunDuration(report),
|
|
361
|
+
totals: {
|
|
362
|
+
total: report.numTotalTests,
|
|
363
|
+
passed: report.numPassedTests,
|
|
364
|
+
failed: report.numFailedTests,
|
|
365
|
+
skipped: report.numPendingTests + report.numTodoTests,
|
|
366
|
+
evalTotal: runCases.length,
|
|
367
|
+
evalPassed: runCases.filter((testCase) => testCase.status === "passed").length,
|
|
368
|
+
evalFailed: runCases.filter((testCase) => testCase.status === "failed").length
|
|
369
|
+
}
|
|
370
|
+
});
|
|
371
|
+
cases.push(...runCases);
|
|
372
|
+
});
|
|
373
|
+
return {
|
|
374
|
+
schemaVersion: REPORT_WORKSPACE_SCHEMA_VERSION,
|
|
375
|
+
runs,
|
|
376
|
+
cases
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
function normalizeWorkspaceInput(input) {
|
|
380
|
+
if (isVitestJsonReportLike(input)) {
|
|
381
|
+
return {
|
|
382
|
+
report: parseVitestJsonReport(input)
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
if (isRecord(input) && "report" in input) {
|
|
386
|
+
return {
|
|
387
|
+
report: parseVitestJsonReport(input.report),
|
|
388
|
+
source: typeof input.source === "string" ? input.source : void 0
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
return {
|
|
392
|
+
report: parseVitestJsonReport(input)
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
function isVitestJsonReportLike(input) {
|
|
396
|
+
return isRecord(input) && Array.isArray(input.testResults);
|
|
397
|
+
}
|
|
398
|
+
function createCaseId(runId, file, assertion) {
|
|
399
|
+
return [runId, file, assertion.location?.line ?? 0, assertion.fullName].join(
|
|
400
|
+
":"
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
function formatDisplayName(assertion) {
|
|
404
|
+
return [...assertion.ancestorTitles, assertion.title].filter((part) => part.length > 0).join(" > ");
|
|
405
|
+
}
|
|
406
|
+
function resolveRunDuration(report) {
|
|
407
|
+
const intervals = report.testResults.map((file) => {
|
|
408
|
+
const start = file.startTime;
|
|
409
|
+
const end = file.endTime;
|
|
410
|
+
if (!isFiniteNumber(start) || !isFiniteNumber(end) || end < start) {
|
|
411
|
+
return void 0;
|
|
412
|
+
}
|
|
413
|
+
return {
|
|
414
|
+
start,
|
|
415
|
+
end
|
|
416
|
+
};
|
|
417
|
+
}).filter(
|
|
418
|
+
(interval) => Boolean(interval)
|
|
419
|
+
);
|
|
420
|
+
if (intervals.length === 0) {
|
|
421
|
+
return void 0;
|
|
422
|
+
}
|
|
423
|
+
return Math.max(...intervals.map((interval) => interval.end)) - Math.min(...intervals.map((interval) => interval.start));
|
|
424
|
+
}
|
|
425
|
+
function isFiniteNumber(value) {
|
|
426
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
427
|
+
}
|
|
428
|
+
function normalizeReportPath(path, workspace) {
|
|
429
|
+
const normalized = path.replace(/\\/g, "/");
|
|
430
|
+
if (!workspace) {
|
|
431
|
+
return normalized;
|
|
432
|
+
}
|
|
433
|
+
const workspacePath = workspace.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
434
|
+
if (normalized !== workspacePath && !normalized.startsWith(`${workspacePath}/`)) {
|
|
435
|
+
return normalized;
|
|
436
|
+
}
|
|
437
|
+
return normalized.slice(workspacePath.length).replace(/^\/+/, "");
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// src/node.ts
|
|
441
|
+
var GLOB_META_PATTERN = /[*?]/;
|
|
442
|
+
var DEFAULT_REPORT_EXTENSIONS = [".json"];
|
|
443
|
+
async function resolveReportFiles(patterns, options = {}) {
|
|
444
|
+
const cwd = options.cwd ?? process.cwd();
|
|
445
|
+
const extensions = normalizeExtensions(
|
|
446
|
+
options.extensions ?? DEFAULT_REPORT_EXTENSIONS
|
|
447
|
+
);
|
|
448
|
+
const files = [];
|
|
449
|
+
for (const pattern of patterns.map((entry) => entry.trim()).filter(Boolean)) {
|
|
450
|
+
if (hasGlob(pattern)) {
|
|
451
|
+
files.push(...await expandGlob(pattern, cwd));
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
const candidate = (0, import_node_path.isAbsolute)(pattern) ? pattern : (0, import_node_path.resolve)(cwd, pattern);
|
|
455
|
+
const candidateStat = await readStat(candidate);
|
|
456
|
+
if (candidateStat?.isDirectory()) {
|
|
457
|
+
files.push(...await listReportFiles(candidate, extensions));
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
460
|
+
files.push(candidate);
|
|
461
|
+
}
|
|
462
|
+
return [...new Set(files)].sort();
|
|
463
|
+
}
|
|
464
|
+
var resolveResultFiles = resolveReportFiles;
|
|
465
|
+
function splitReportInput(value) {
|
|
466
|
+
return (value ?? "").split(/\r?\n/).map((entry) => entry.trim()).filter(Boolean);
|
|
467
|
+
}
|
|
468
|
+
var splitResultsInput = splitReportInput;
|
|
469
|
+
async function readVitestJsonReportFile(resultFile) {
|
|
470
|
+
try {
|
|
471
|
+
return parseVitestJsonReport(
|
|
472
|
+
JSON.parse(await (0, import_promises.readFile)(resultFile, "utf8"))
|
|
473
|
+
);
|
|
474
|
+
} catch (error) {
|
|
475
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
476
|
+
throw new Error(
|
|
477
|
+
`Failed to read eval result file ${resultFile}: ${message}`
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
async function readReportWorkspace(patterns, options = {}) {
|
|
482
|
+
const cwd = options.cwd ?? process.cwd();
|
|
483
|
+
const resultFiles = await resolveReportFiles(patterns, { ...options, cwd });
|
|
484
|
+
if (resultFiles.length === 0) {
|
|
485
|
+
throw new Error(`No eval result files matched: ${patterns.join(", ")}`);
|
|
486
|
+
}
|
|
487
|
+
const reports = await Promise.all(
|
|
488
|
+
resultFiles.map(async (resultFile) => ({
|
|
489
|
+
report: await readVitestJsonReportFile(resultFile),
|
|
490
|
+
source: displayReportSource(resultFile, cwd)
|
|
491
|
+
}))
|
|
492
|
+
);
|
|
493
|
+
return {
|
|
494
|
+
resultFiles,
|
|
495
|
+
workspace: collectReportWorkspace(reports, {
|
|
496
|
+
workspace: options.workspace
|
|
497
|
+
})
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
function displayReportSource(resultFile, cwd) {
|
|
501
|
+
const relativeSource = normalizePath((0, import_node_path.relative)((0, import_node_path.resolve)(cwd), resultFile));
|
|
502
|
+
if (relativeSource === "" || relativeSource === "." || relativeSource.startsWith("../")) {
|
|
503
|
+
return resultFile;
|
|
504
|
+
}
|
|
505
|
+
return relativeSource;
|
|
506
|
+
}
|
|
507
|
+
function hasGlob(pattern) {
|
|
508
|
+
return GLOB_META_PATTERN.test(pattern);
|
|
509
|
+
}
|
|
510
|
+
async function expandGlob(pattern, cwd) {
|
|
511
|
+
const normalizedPattern = normalizeGlobPattern(pattern);
|
|
512
|
+
const absolutePattern = (0, import_node_path.isAbsolute)(pattern);
|
|
513
|
+
const base = globBase(normalizedPattern);
|
|
514
|
+
const basePath = absolutePattern ? base || import_node_path.sep : (0, import_node_path.resolve)(cwd, base || ".");
|
|
515
|
+
const regex = globToRegExp(normalizedPattern);
|
|
516
|
+
const matches = [];
|
|
517
|
+
for (const file of await listFiles(basePath)) {
|
|
518
|
+
const normalizedFile = normalizePath(file);
|
|
519
|
+
const candidate = absolutePattern ? normalizedFile : normalizePath((0, import_node_path.relative)((0, import_node_path.resolve)(cwd), file));
|
|
520
|
+
if (regex.test(candidate)) {
|
|
521
|
+
matches.push(file);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return matches;
|
|
525
|
+
}
|
|
526
|
+
function globBase(pattern) {
|
|
527
|
+
const segments = pattern.split("/");
|
|
528
|
+
const baseSegments = [];
|
|
529
|
+
for (const segment of segments) {
|
|
530
|
+
if (hasGlob(segment)) {
|
|
531
|
+
break;
|
|
532
|
+
}
|
|
533
|
+
baseSegments.push(segment);
|
|
534
|
+
}
|
|
535
|
+
return baseSegments.join("/");
|
|
536
|
+
}
|
|
537
|
+
async function listReportFiles(directory, extensions) {
|
|
538
|
+
const files = await listFiles(directory);
|
|
539
|
+
return files.filter((file) => extensions.has((0, import_node_path.extname)(file).toLowerCase()));
|
|
540
|
+
}
|
|
541
|
+
async function listFiles(directory) {
|
|
542
|
+
const entries = await readDirectory(directory);
|
|
543
|
+
if (!entries) {
|
|
544
|
+
return [];
|
|
545
|
+
}
|
|
546
|
+
const files = [];
|
|
547
|
+
for (const entry of entries) {
|
|
548
|
+
const child = (0, import_node_path.resolve)(directory, entry.name);
|
|
549
|
+
if (entry.isDirectory()) {
|
|
550
|
+
files.push(...await listFiles(child));
|
|
551
|
+
} else if (entry.isFile()) {
|
|
552
|
+
files.push(child);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return files;
|
|
556
|
+
}
|
|
557
|
+
async function readDirectory(directory) {
|
|
558
|
+
try {
|
|
559
|
+
return await (0, import_promises.readdir)(directory, { withFileTypes: true });
|
|
560
|
+
} catch {
|
|
561
|
+
return void 0;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
async function readStat(path) {
|
|
565
|
+
try {
|
|
566
|
+
return await (0, import_promises.stat)(path);
|
|
567
|
+
} catch {
|
|
568
|
+
return void 0;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
function globToRegExp(pattern) {
|
|
572
|
+
let source = "";
|
|
573
|
+
for (let index = 0; index < pattern.length; index += 1) {
|
|
574
|
+
const char = pattern[index];
|
|
575
|
+
const next = pattern[index + 1];
|
|
576
|
+
if (char === "*" && next === "*") {
|
|
577
|
+
const following = pattern[index + 2];
|
|
578
|
+
if (following === "/") {
|
|
579
|
+
source += "(?:.*/)?";
|
|
580
|
+
index += 2;
|
|
581
|
+
} else {
|
|
582
|
+
source += ".*";
|
|
583
|
+
index += 1;
|
|
584
|
+
}
|
|
585
|
+
continue;
|
|
586
|
+
}
|
|
587
|
+
if (char === "*") {
|
|
588
|
+
source += "[^/]*";
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
if (char === "?") {
|
|
592
|
+
source += "[^/]";
|
|
593
|
+
continue;
|
|
594
|
+
}
|
|
595
|
+
source += escapeRegExp(char ?? "");
|
|
596
|
+
}
|
|
597
|
+
return new RegExp(`^${source}$`);
|
|
598
|
+
}
|
|
599
|
+
function escapeRegExp(value) {
|
|
600
|
+
return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
|
|
601
|
+
}
|
|
602
|
+
function normalizePath(path) {
|
|
603
|
+
return path.replace(/\\/g, "/");
|
|
604
|
+
}
|
|
605
|
+
function normalizeGlobPattern(pattern) {
|
|
606
|
+
const normalizedPattern = normalizePath(pattern);
|
|
607
|
+
if ((0, import_node_path.isAbsolute)(pattern)) {
|
|
608
|
+
return normalizedPattern;
|
|
609
|
+
}
|
|
610
|
+
return normalizedPattern.replace(/^(\.\/)+/, "");
|
|
611
|
+
}
|
|
612
|
+
function normalizeExtensions(extensions) {
|
|
613
|
+
return new Set(
|
|
614
|
+
extensions.map(
|
|
615
|
+
(extension) => extension.startsWith(".") ? extension.toLowerCase() : `.${extension.toLowerCase()}`
|
|
616
|
+
)
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
620
|
+
0 && (module.exports = {
|
|
621
|
+
readReportWorkspace,
|
|
622
|
+
readVitestJsonReportFile,
|
|
623
|
+
resolveReportFiles,
|
|
624
|
+
resolveResultFiles,
|
|
625
|
+
splitReportInput,
|
|
626
|
+
splitResultsInput
|
|
627
|
+
});
|
|
628
|
+
//# sourceMappingURL=node.js.map
|