@elench/testkit 0.1.80 → 0.1.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -35
- package/lib/cli/args.mjs +2 -14
- package/lib/cli/args.test.mjs +1 -17
- package/lib/cli/command-helpers.mjs +1 -20
- package/lib/cli/entrypoint.mjs +0 -4
- package/lib/cli/presentation/colors.mjs +1 -1
- package/lib/cli/presentation/failure-presentation.mjs +4 -4
- package/lib/cli/presentation/run-reporter.mjs +23 -9
- package/lib/cli/presentation/run-reporter.test.mjs +12 -6
- package/lib/cli/presentation/summary-box.test.mjs +4 -4
- package/lib/cli/viewer.mjs +18 -19
- package/lib/config/index.mjs +6 -6
- package/lib/config/runtime.mjs +8 -8
- package/lib/config-api/index.d.ts +4 -4
- package/lib/package.test.mjs +4 -4
- package/lib/{known-failures → regressions}/github.mjs +36 -78
- package/lib/regressions/github.test.mjs +324 -0
- package/lib/regressions/index.d.ts +189 -0
- package/lib/{known-failures → regressions}/index.mjs +90 -93
- package/lib/{known-failures → regressions}/index.test.mjs +37 -48
- package/lib/runner/formatting.mjs +49 -34
- package/lib/runner/formatting.test.mjs +16 -15
- package/lib/runner/metadata.mjs +1 -1
- package/lib/runner/orchestrator.mjs +7 -9
- package/lib/runner/regressions.mjs +304 -0
- package/lib/runner/{triage.test.mjs → regressions.test.mjs} +50 -36
- package/lib/runner/reporting.mjs +2 -2
- package/lib/runner/reporting.test.mjs +2 -2
- package/lib/runner/run-finalization.mjs +18 -30
- package/lib/runner/template-steps.mjs +2 -2
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/package.json +8 -8
- package/lib/cli/commands/known-failures/render.mjs +0 -19
- package/lib/cli/commands/known-failures/validate.mjs +0 -20
- package/lib/cli/known-failures.mjs +0 -164
- package/lib/known-failures/github.test.mjs +0 -512
- package/lib/known-failures/index.d.ts +0 -192
- package/lib/runner/triage.mjs +0 -221
- /package/lib/{known-failures → regressions}/github-cache.mjs +0 -0
- /package/lib/{known-failures → regressions}/github-transport.mjs +0 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import {
|
|
2
|
+
buildRegressionFileIdentity,
|
|
3
|
+
findMatchingRegressionEntries,
|
|
4
|
+
loadRegressionCatalogConfig,
|
|
5
|
+
} from "../regressions/index.mjs";
|
|
6
|
+
|
|
7
|
+
export { loadRegressionCatalogConfig };
|
|
8
|
+
|
|
9
|
+
export function applyRegressionAnalysisToArtifacts(
|
|
10
|
+
runArtifact,
|
|
11
|
+
statusArtifact,
|
|
12
|
+
regressionCatalog,
|
|
13
|
+
regressionSync
|
|
14
|
+
) {
|
|
15
|
+
const runEntries = extractRunFileEntries(runArtifact);
|
|
16
|
+
const statusEntries = extractStatusFileEntries(statusArtifact);
|
|
17
|
+
const fileSummaries = new Map();
|
|
18
|
+
const syncById = new Map((regressionSync?.entries || []).map((entry) => [entry.id, entry]));
|
|
19
|
+
const staleEntryIds = new Set();
|
|
20
|
+
const newRegressionDrafts = [];
|
|
21
|
+
const fixedRegressionDrafts = [];
|
|
22
|
+
|
|
23
|
+
for (const entry of [...runEntries, ...statusEntries]) {
|
|
24
|
+
const key = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
|
|
25
|
+
if (!fileSummaries.has(key)) {
|
|
26
|
+
fileSummaries.set(key, {
|
|
27
|
+
service: entry.service,
|
|
28
|
+
type: entry.type,
|
|
29
|
+
path: entry.path,
|
|
30
|
+
status: entry.status,
|
|
31
|
+
error: entry.error || null,
|
|
32
|
+
failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const diagnosesByFileKey = new Map();
|
|
38
|
+
|
|
39
|
+
for (const fileSummary of fileSummaries.values()) {
|
|
40
|
+
const diagnosis = buildFileDiagnosis(fileSummary, regressionCatalog, syncById);
|
|
41
|
+
const fileKey = buildRegressionFileIdentity(
|
|
42
|
+
fileSummary.service,
|
|
43
|
+
fileSummary.type,
|
|
44
|
+
fileSummary.path
|
|
45
|
+
);
|
|
46
|
+
diagnosesByFileKey.set(fileKey, diagnosis);
|
|
47
|
+
for (const entry of diagnosis.entries) {
|
|
48
|
+
if (entry.catalogFindings.length > 0) {
|
|
49
|
+
staleEntryIds.add(entry.id);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (diagnosis.status === "new_regression") {
|
|
53
|
+
newRegressionDrafts.push(buildNewRegressionDraft(fileSummary, regressionCatalog));
|
|
54
|
+
}
|
|
55
|
+
if (diagnosis.status === "fixed_known_regression") {
|
|
56
|
+
for (const entry of diagnosis.entries) {
|
|
57
|
+
fixedRegressionDrafts.push({
|
|
58
|
+
id: entry.id,
|
|
59
|
+
issue: entry.issue,
|
|
60
|
+
summary: entry.summary,
|
|
61
|
+
suggestedAction: "review-for-removal",
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const entry of [...runEntries, ...statusEntries]) {
|
|
68
|
+
const fileKey = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
|
|
69
|
+
const diagnosis = diagnosesByFileKey.get(fileKey);
|
|
70
|
+
if (!diagnosis) continue;
|
|
71
|
+
setEntryDiagnosis(entry, diagnosis);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const summaryTests = statusArtifact?.tests || runEntries;
|
|
75
|
+
const report = buildRegressionReport(
|
|
76
|
+
summaryTests,
|
|
77
|
+
regressionCatalog,
|
|
78
|
+
regressionSync,
|
|
79
|
+
diagnosesByFileKey,
|
|
80
|
+
staleEntryIds,
|
|
81
|
+
newRegressionDrafts,
|
|
82
|
+
fixedRegressionDrafts
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
runArtifact.regressions = report;
|
|
86
|
+
if (statusArtifact) {
|
|
87
|
+
statusArtifact.regressions = report;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { runArtifact, statusArtifact, regressionReport: report };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function buildFileDiagnosis(fileSummary, regressionCatalog, syncById) {
|
|
94
|
+
const matchedEntries = regressionCatalog
|
|
95
|
+
? findMatchingRegressionEntries(regressionCatalog, fileSummary)
|
|
96
|
+
: [];
|
|
97
|
+
const status = resolveDiagnosisStatus(fileSummary.status, matchedEntries.length);
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
status,
|
|
101
|
+
classifications: [...new Set(matchedEntries.map((entry) => entry.classification))].sort(),
|
|
102
|
+
entries: matchedEntries.map((entry) => toDiagnosisEntry(entry, syncById.get(entry.id) || null)),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function resolveDiagnosisStatus(fileStatus, matchCount) {
|
|
107
|
+
if (fileStatus === "failed") {
|
|
108
|
+
return matchCount > 0 ? "known_regression" : "new_regression";
|
|
109
|
+
}
|
|
110
|
+
if (fileStatus === "passed" && matchCount > 0) {
|
|
111
|
+
return "fixed_known_regression";
|
|
112
|
+
}
|
|
113
|
+
if (matchCount > 0) {
|
|
114
|
+
return "tracked_regression_not_executed";
|
|
115
|
+
}
|
|
116
|
+
return "not_applicable";
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function toDiagnosisEntry(entry, syncEntry) {
|
|
120
|
+
return {
|
|
121
|
+
id: entry.id,
|
|
122
|
+
classification: entry.classification,
|
|
123
|
+
issue: entry.issue,
|
|
124
|
+
summary: entry.summary,
|
|
125
|
+
cause: entry.cause,
|
|
126
|
+
lastReviewedAt: entry.lastReviewedAt,
|
|
127
|
+
github: syncEntry?.github || null,
|
|
128
|
+
syncStatus: syncEntry?.status || null,
|
|
129
|
+
catalogFindings: Array.isArray(syncEntry?.findings) ? syncEntry.findings : [],
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function buildRegressionReport(
|
|
134
|
+
tests,
|
|
135
|
+
regressionCatalog,
|
|
136
|
+
regressionSync,
|
|
137
|
+
diagnosesByFileKey,
|
|
138
|
+
staleEntryIds,
|
|
139
|
+
newRegressionDrafts,
|
|
140
|
+
fixedRegressionDrafts
|
|
141
|
+
) {
|
|
142
|
+
const summary = {
|
|
143
|
+
newRegressions: 0,
|
|
144
|
+
knownRegressions: 0,
|
|
145
|
+
fixedKnownRegressions: 0,
|
|
146
|
+
catalogStale: staleEntryIds.size,
|
|
147
|
+
catalogSyncUnavailable: (regressionSync?.summary?.byCode?.validation_unavailable || 0) > 0,
|
|
148
|
+
usedStaleCache: (regressionSync?.summary?.byCode?.used_stale_cache || 0) > 0,
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
for (const test of tests) {
|
|
152
|
+
const diagnosis = test.diagnosis || diagnosesByFileKey.get(
|
|
153
|
+
buildRegressionFileIdentity(test.service, test.type, test.path)
|
|
154
|
+
);
|
|
155
|
+
switch (diagnosis?.status) {
|
|
156
|
+
case "new_regression":
|
|
157
|
+
summary.newRegressions += 1;
|
|
158
|
+
break;
|
|
159
|
+
case "known_regression":
|
|
160
|
+
summary.knownRegressions += 1;
|
|
161
|
+
break;
|
|
162
|
+
case "fixed_known_regression":
|
|
163
|
+
summary.fixedKnownRegressions += 1;
|
|
164
|
+
break;
|
|
165
|
+
default:
|
|
166
|
+
break;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return {
|
|
171
|
+
summary,
|
|
172
|
+
catalog: {
|
|
173
|
+
configured: Boolean(regressionCatalog),
|
|
174
|
+
entryCount: regressionCatalog?.entries?.length || 0,
|
|
175
|
+
staleEntries: (regressionSync?.entries || []).filter((entry) =>
|
|
176
|
+
Array.isArray(entry.findings) && entry.findings.length > 0
|
|
177
|
+
),
|
|
178
|
+
findings: regressionSync?.findings || [],
|
|
179
|
+
sync: {
|
|
180
|
+
mode: regressionSync?.mode || null,
|
|
181
|
+
checkedAt: regressionSync?.checkedAt || null,
|
|
182
|
+
usedStaleCache: summary.usedStaleCache,
|
|
183
|
+
unavailable: summary.catalogSyncUnavailable,
|
|
184
|
+
},
|
|
185
|
+
},
|
|
186
|
+
drafts: {
|
|
187
|
+
newRegressions: newRegressionDrafts,
|
|
188
|
+
fixedRegressions: dedupeDraftsById(fixedRegressionDrafts),
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function buildNewRegressionDraft(fileSummary, regressionCatalog) {
|
|
194
|
+
return {
|
|
195
|
+
id: suggestRegressionId(fileSummary),
|
|
196
|
+
classification: suggestRegressionClassification(fileSummary),
|
|
197
|
+
issue: {
|
|
198
|
+
repo: regressionCatalog?.issueRepo || null,
|
|
199
|
+
number: null,
|
|
200
|
+
},
|
|
201
|
+
summary: suggestRegressionSummary(fileSummary),
|
|
202
|
+
cause: suggestRegressionCause(fileSummary),
|
|
203
|
+
fingerprints: [
|
|
204
|
+
{
|
|
205
|
+
service: fileSummary.service,
|
|
206
|
+
type: fileSummary.type,
|
|
207
|
+
path: fileSummary.path,
|
|
208
|
+
...(fileSummary.failureDetails?.[0]?.key ? { failureKey: fileSummary.failureDetails[0].key } : {}),
|
|
209
|
+
},
|
|
210
|
+
],
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function suggestRegressionId(fileSummary) {
|
|
215
|
+
const base = `${fileSummary.service}-${fileSummary.type}-${fileSummary.path}`
|
|
216
|
+
.toLowerCase()
|
|
217
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
218
|
+
.replace(/^-+|-+$/g, "");
|
|
219
|
+
const detail = fileSummary.failureDetails?.[0]?.key
|
|
220
|
+
? `-${String(fileSummary.failureDetails[0].key).toLowerCase().replace(/[^a-z0-9]+/g, "-")}`
|
|
221
|
+
: "";
|
|
222
|
+
return `${base}${detail}`.replace(/-+/g, "-").slice(0, 96);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function suggestRegressionClassification(fileSummary) {
|
|
226
|
+
const message = [
|
|
227
|
+
fileSummary.error,
|
|
228
|
+
...((fileSummary.failureDetails || []).map((detail) => detail?.message || detail?.title || "")),
|
|
229
|
+
]
|
|
230
|
+
.filter(Boolean)
|
|
231
|
+
.join(" ")
|
|
232
|
+
.toLowerCase();
|
|
233
|
+
|
|
234
|
+
if (
|
|
235
|
+
message.includes("timed out") ||
|
|
236
|
+
message.includes("already in use") ||
|
|
237
|
+
message.includes("never becomes ready") ||
|
|
238
|
+
message.includes("runtime error")
|
|
239
|
+
) {
|
|
240
|
+
return "infra";
|
|
241
|
+
}
|
|
242
|
+
return "product_bug";
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function suggestRegressionSummary(fileSummary) {
|
|
246
|
+
const primaryDetail = fileSummary.failureDetails?.[0];
|
|
247
|
+
if (primaryDetail?.message) return String(primaryDetail.message).trim().slice(0, 160);
|
|
248
|
+
if (primaryDetail?.title) return String(primaryDetail.title).trim().slice(0, 160);
|
|
249
|
+
if (fileSummary.error) return String(fileSummary.error).trim().slice(0, 160);
|
|
250
|
+
return `${fileSummary.type} regression in ${fileSummary.path}`;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function suggestRegressionCause(fileSummary) {
|
|
254
|
+
const primaryDetail = fileSummary.failureDetails?.[0];
|
|
255
|
+
if (primaryDetail?.response?.bodyPreview) {
|
|
256
|
+
return `Observed response preview: ${String(primaryDetail.response.bodyPreview).slice(0, 200)}`;
|
|
257
|
+
}
|
|
258
|
+
if (fileSummary.error) {
|
|
259
|
+
return `Observed failure: ${String(fileSummary.error).slice(0, 200)}`;
|
|
260
|
+
}
|
|
261
|
+
return "Investigate the observed failure and replace this draft cause with the underlying technical root cause.";
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function dedupeDraftsById(entries) {
|
|
265
|
+
const map = new Map();
|
|
266
|
+
for (const entry of entries) {
|
|
267
|
+
if (!map.has(entry.id)) map.set(entry.id, entry);
|
|
268
|
+
}
|
|
269
|
+
return [...map.values()];
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function extractRunFileEntries(runArtifact) {
|
|
273
|
+
const entries = [];
|
|
274
|
+
|
|
275
|
+
for (const service of runArtifact.services || []) {
|
|
276
|
+
for (const suite of service.suites || []) {
|
|
277
|
+
for (const file of suite.files || []) {
|
|
278
|
+
entries.push({
|
|
279
|
+
target: file,
|
|
280
|
+
service: service.name,
|
|
281
|
+
type: suite.type,
|
|
282
|
+
path: file.path,
|
|
283
|
+
status: file.status,
|
|
284
|
+
error: file.error || null,
|
|
285
|
+
failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return entries;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function extractStatusFileEntries(statusArtifact) {
|
|
295
|
+
return statusArtifact?.tests || [];
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function setEntryDiagnosis(entry, diagnosis) {
|
|
299
|
+
if (entry?.target) {
|
|
300
|
+
entry.target.diagnosis = diagnosis;
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
entry.diagnosis = diagnosis;
|
|
304
|
+
}
|
|
@@ -1,27 +1,24 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { normalizeRegressionCatalogDocument } from "../regressions/index.mjs";
|
|
3
|
+
import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
|
|
4
4
|
|
|
5
|
-
describe("runner
|
|
6
|
-
it("
|
|
7
|
-
const
|
|
5
|
+
describe("runner regressions", () => {
|
|
6
|
+
it("classifies known failed regressions and enriches both artifacts", () => {
|
|
7
|
+
const regressionCatalog = normalizeRegressionCatalogDocument({
|
|
8
8
|
schemaVersion: 1,
|
|
9
9
|
issueRepo: "acme/repo",
|
|
10
10
|
entries: [
|
|
11
11
|
{
|
|
12
12
|
id: "bad-message",
|
|
13
|
-
title: "Bad message bug",
|
|
14
13
|
classification: "product_bug",
|
|
15
|
-
state: "open",
|
|
16
14
|
issue: {
|
|
17
15
|
repo: "acme/repo",
|
|
18
16
|
number: 12,
|
|
19
|
-
url: "https://github.com/acme/repo/issues/12",
|
|
20
17
|
},
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
lastReviewedAt: "2026-04
|
|
24
|
-
|
|
18
|
+
summary: "API returns the wrong message",
|
|
19
|
+
cause: "The endpoint payload is wrong.",
|
|
20
|
+
lastReviewedAt: "2026-05-04",
|
|
21
|
+
fingerprints: [
|
|
25
22
|
{
|
|
26
23
|
service: "api",
|
|
27
24
|
type: "int",
|
|
@@ -80,36 +77,49 @@ describe("runner triage", () => {
|
|
|
80
77
|
],
|
|
81
78
|
};
|
|
82
79
|
|
|
83
|
-
const enriched =
|
|
84
|
-
|
|
85
|
-
|
|
80
|
+
const enriched = applyRegressionAnalysisToArtifacts(
|
|
81
|
+
runArtifact,
|
|
82
|
+
statusArtifact,
|
|
83
|
+
regressionCatalog,
|
|
84
|
+
{
|
|
85
|
+
entries: [
|
|
86
|
+
{
|
|
87
|
+
id: "bad-message",
|
|
88
|
+
status: "open_and_failing",
|
|
89
|
+
github: {
|
|
90
|
+
state: "OPEN",
|
|
91
|
+
url: "https://github.com/acme/repo/issues/12",
|
|
92
|
+
cached: false,
|
|
93
|
+
},
|
|
94
|
+
findings: [],
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
findings: [],
|
|
98
|
+
}
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
expect(enriched.statusArtifact.tests[0].diagnosis).toMatchObject({
|
|
102
|
+
status: "known_regression",
|
|
86
103
|
classifications: ["product_bug"],
|
|
87
104
|
});
|
|
88
|
-
expect(enriched.runArtifact.services[0].suites[0].files[0].
|
|
105
|
+
expect(enriched.runArtifact.services[0].suites[0].files[0].diagnosis.entries[0]).toMatchObject({
|
|
89
106
|
id: "bad-message",
|
|
90
107
|
issue: {
|
|
91
108
|
number: 12,
|
|
92
109
|
},
|
|
93
110
|
});
|
|
94
|
-
expect(enriched.statusArtifact.
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
},
|
|
102
|
-
},
|
|
103
|
-
entries: {
|
|
104
|
-
total: 1,
|
|
105
|
-
matchedByFailedTests: 1,
|
|
106
|
-
unmatched: 0,
|
|
107
|
-
},
|
|
111
|
+
expect(enriched.statusArtifact.regressions.summary).toEqual({
|
|
112
|
+
newRegressions: 0,
|
|
113
|
+
knownRegressions: 1,
|
|
114
|
+
fixedKnownRegressions: 0,
|
|
115
|
+
catalogStale: 0,
|
|
116
|
+
catalogSyncUnavailable: false,
|
|
117
|
+
usedStaleCache: false,
|
|
108
118
|
});
|
|
109
119
|
});
|
|
110
120
|
|
|
111
|
-
it("marks unmatched failed tests as
|
|
112
|
-
const enriched =
|
|
121
|
+
it("marks unmatched failed tests as new regressions and emits drafts", () => {
|
|
122
|
+
const enriched = applyRegressionAnalysisToArtifacts(
|
|
113
123
|
{
|
|
114
124
|
services: [
|
|
115
125
|
{
|
|
@@ -140,15 +150,19 @@ describe("runner triage", () => {
|
|
|
140
150
|
},
|
|
141
151
|
],
|
|
142
152
|
},
|
|
143
|
-
|
|
153
|
+
normalizeRegressionCatalogDocument({
|
|
144
154
|
schemaVersion: 1,
|
|
145
155
|
entries: [],
|
|
146
|
-
})
|
|
156
|
+
}),
|
|
157
|
+
null
|
|
147
158
|
);
|
|
148
159
|
|
|
149
|
-
expect(enriched.statusArtifact.tests[0].
|
|
150
|
-
status: "
|
|
160
|
+
expect(enriched.statusArtifact.tests[0].diagnosis).toEqual({
|
|
161
|
+
status: "new_regression",
|
|
162
|
+
classifications: [],
|
|
151
163
|
entries: [],
|
|
152
164
|
});
|
|
165
|
+
expect(enriched.statusArtifact.regressions.summary.newRegressions).toBe(1);
|
|
166
|
+
expect(enriched.statusArtifact.regressions.drafts.newRegressions).toHaveLength(1);
|
|
153
167
|
});
|
|
154
168
|
});
|
package/lib/runner/reporting.mjs
CHANGED
|
@@ -80,7 +80,7 @@ export function buildStatusArtifact({
|
|
|
80
80
|
scope.serviceFilter === null;
|
|
81
81
|
|
|
82
82
|
return {
|
|
83
|
-
schemaVersion:
|
|
83
|
+
schemaVersion: 7,
|
|
84
84
|
source: "testkit",
|
|
85
85
|
notice: "Generated file. Do not edit manually.",
|
|
86
86
|
product: {
|
|
@@ -134,7 +134,7 @@ export function buildRunArtifact({
|
|
|
134
134
|
const dbBackend = summarizeDbBackend(results);
|
|
135
135
|
|
|
136
136
|
return {
|
|
137
|
-
schemaVersion:
|
|
137
|
+
schemaVersion: 9,
|
|
138
138
|
source: "testkit",
|
|
139
139
|
generatedAt: new Date(finishedAt).toISOString(),
|
|
140
140
|
product: {
|
|
@@ -79,7 +79,7 @@ describe("runner reporting", () => {
|
|
|
79
79
|
});
|
|
80
80
|
|
|
81
81
|
expect(artifact.product.name).toBe("my-product");
|
|
82
|
-
expect(artifact.schemaVersion).toBe(
|
|
82
|
+
expect(artifact.schemaVersion).toBe(9);
|
|
83
83
|
expect(artifact.run).toMatchObject({
|
|
84
84
|
workers: 2,
|
|
85
85
|
fileTimeoutSeconds: 60,
|
|
@@ -233,7 +233,7 @@ describe("runner reporting", () => {
|
|
|
233
233
|
});
|
|
234
234
|
|
|
235
235
|
expect(status).toEqual({
|
|
236
|
-
schemaVersion:
|
|
236
|
+
schemaVersion: 7,
|
|
237
237
|
source: "testkit",
|
|
238
238
|
notice: "Generated file. Do not edit manually.",
|
|
239
239
|
product: {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
|
|
3
3
|
import { writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
|
|
4
4
|
import { summarizeDbBackend } from "./results.mjs";
|
|
5
5
|
import { formatError } from "./formatting.mjs";
|
|
6
6
|
import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
|
|
7
7
|
import { loadHistory, saveHistory, updateHistoryFromRunArtifact } from "../history/index.mjs";
|
|
8
|
-
import {
|
|
8
|
+
import { shouldFailRegressionSync, validateRegressionIssues } from "../regressions/github.mjs";
|
|
9
9
|
|
|
10
10
|
export async function finalizeRunArtifacts({
|
|
11
11
|
productDir,
|
|
@@ -20,8 +20,8 @@ export async function finalizeRunArtifacts({
|
|
|
20
20
|
metadata,
|
|
21
21
|
logRegistry,
|
|
22
22
|
setupRegistry,
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
regressionCatalog,
|
|
24
|
+
regressionSyncConfig,
|
|
25
25
|
telemetry,
|
|
26
26
|
reporter,
|
|
27
27
|
writeStatus,
|
|
@@ -60,24 +60,19 @@ export async function finalizeRunArtifacts({
|
|
|
60
60
|
metadata,
|
|
61
61
|
})
|
|
62
62
|
: null;
|
|
63
|
-
const
|
|
64
|
-
const knownFailureIssueValidation = await validateKnownFailureIssues({
|
|
63
|
+
const regressionSync = await validateRegressionIssues({
|
|
65
64
|
productDir,
|
|
66
|
-
document:
|
|
67
|
-
runArtifact
|
|
68
|
-
statusArtifact
|
|
69
|
-
config:
|
|
65
|
+
document: regressionCatalog,
|
|
66
|
+
runArtifact,
|
|
67
|
+
statusArtifact,
|
|
68
|
+
config: regressionSyncConfig,
|
|
70
69
|
gitMetadata: metadata.git,
|
|
71
70
|
});
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
attachKnownFailureIssueValidation(
|
|
78
|
-
enrichedArtifacts.runArtifact,
|
|
79
|
-
enrichedArtifacts.statusArtifact,
|
|
80
|
-
knownFailureIssueValidation
|
|
71
|
+
const enrichedArtifacts = applyRegressionAnalysisToArtifacts(
|
|
72
|
+
runArtifact,
|
|
73
|
+
statusArtifact,
|
|
74
|
+
regressionCatalog,
|
|
75
|
+
regressionSync
|
|
81
76
|
);
|
|
82
77
|
|
|
83
78
|
writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
|
|
@@ -91,14 +86,15 @@ export async function finalizeRunArtifacts({
|
|
|
91
86
|
);
|
|
92
87
|
saveHistory(productDir, nextHistory);
|
|
93
88
|
|
|
94
|
-
reporter?.runSummary?.(results, finishedAt - startedAt,
|
|
89
|
+
reporter?.runSummary?.(results, finishedAt - startedAt, enrichedArtifacts.regressionReport);
|
|
95
90
|
await reportTelemetry(telemetry, enrichedArtifacts.runArtifact, reporter);
|
|
96
91
|
|
|
97
92
|
return {
|
|
98
93
|
runArtifact: enrichedArtifacts.runArtifact,
|
|
99
94
|
statusArtifact: enrichedArtifacts.statusArtifact,
|
|
100
|
-
|
|
101
|
-
|
|
95
|
+
regressionReport: enrichedArtifacts.regressionReport,
|
|
96
|
+
regressionSync,
|
|
97
|
+
shouldFailRegressionSync: shouldFailRegressionSync(regressionSync),
|
|
102
98
|
};
|
|
103
99
|
}
|
|
104
100
|
|
|
@@ -122,11 +118,3 @@ async function reportTelemetry(telemetry, artifact, reporter = null) {
|
|
|
122
118
|
reporter?.telemetry?.(`Telemetry: upload failed (${formatError(error)})`);
|
|
123
119
|
}
|
|
124
120
|
}
|
|
125
|
-
|
|
126
|
-
function attachKnownFailureIssueValidation(runArtifact, statusArtifact, validation) {
|
|
127
|
-
if (!validation) return;
|
|
128
|
-
runArtifact.knownFailuresIssueValidation = validation;
|
|
129
|
-
if (statusArtifact) {
|
|
130
|
-
statusArtifact.knownFailuresIssueValidation = validation;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
@@ -35,7 +35,7 @@ const PLAYWRIGHT_ENTRY = path.join(PACKAGE_ROOT, "lib", "playwright", "index.mjs
|
|
|
35
35
|
const RUNTIME_ENTRY = path.join(PACKAGE_ROOT, "lib", "runtime", "index.mjs");
|
|
36
36
|
const VITEST_ENTRY = path.join(PACKAGE_ROOT, "lib", "vitest", "index.mjs");
|
|
37
37
|
const DISCOVERY_ENTRY = path.join(PACKAGE_ROOT, "lib", "discovery", "index.mjs");
|
|
38
|
-
const
|
|
38
|
+
const REGRESSIONS_ENTRY = path.join(PACKAGE_ROOT, "lib", "regressions", "index.mjs");
|
|
39
39
|
const MODULE_RUNNER_ENTRY = path.join(
|
|
40
40
|
PACKAGE_ROOT,
|
|
41
41
|
"lib",
|
|
@@ -263,7 +263,7 @@ function resolvePackageSubpath(specifier) {
|
|
|
263
263
|
if (subpath === "/runtime") return RUNTIME_ENTRY;
|
|
264
264
|
if (subpath === "/vitest") return VITEST_ENTRY;
|
|
265
265
|
if (subpath === "/discovery") return DISCOVERY_ENTRY;
|
|
266
|
-
if (subpath === "/
|
|
266
|
+
if (subpath === "/regressions") return REGRESSIONS_ENTRY;
|
|
267
267
|
|
|
268
268
|
throw new Error(`Unsupported @elench/testkit import "${specifier}" while loading template step`);
|
|
269
269
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit-bridge",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.81",
|
|
4
4
|
"description": "Browser bridge helpers for testkit",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"typecheck": "tsc -p tsconfig.json --noEmit"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@elench/testkit-protocol": "0.1.
|
|
25
|
+
"@elench/testkit-protocol": "0.1.81"
|
|
26
26
|
},
|
|
27
27
|
"private": false
|
|
28
28
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.81",
|
|
4
4
|
"description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"workspaces": [
|
|
@@ -40,9 +40,9 @@
|
|
|
40
40
|
"types": "./lib/discovery/index.d.ts",
|
|
41
41
|
"default": "./lib/discovery/index.mjs"
|
|
42
42
|
},
|
|
43
|
-
"./
|
|
44
|
-
"types": "./lib/
|
|
45
|
-
"default": "./lib/
|
|
43
|
+
"./regressions": {
|
|
44
|
+
"types": "./lib/regressions/index.d.ts",
|
|
45
|
+
"default": "./lib/regressions/index.mjs"
|
|
46
46
|
},
|
|
47
47
|
"./package.json": "./package.json"
|
|
48
48
|
},
|
|
@@ -81,10 +81,10 @@
|
|
|
81
81
|
},
|
|
82
82
|
"dependencies": {
|
|
83
83
|
"@babel/code-frame": "^7.29.0",
|
|
84
|
-
"@elench/next-analysis": "0.1.
|
|
85
|
-
"@elench/testkit-bridge": "0.1.
|
|
86
|
-
"@elench/testkit-protocol": "0.1.
|
|
87
|
-
"@elench/ts-analysis": "0.1.
|
|
84
|
+
"@elench/next-analysis": "0.1.81",
|
|
85
|
+
"@elench/testkit-bridge": "0.1.81",
|
|
86
|
+
"@elench/testkit-protocol": "0.1.81",
|
|
87
|
+
"@elench/ts-analysis": "0.1.81",
|
|
88
88
|
"@oclif/core": "^4.10.6",
|
|
89
89
|
"esbuild": "^0.25.11",
|
|
90
90
|
"execa": "^9.5.0",
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { Command } from "@oclif/core";
|
|
2
|
-
import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
|
|
3
|
-
import { runKnownFailuresRenderCommand } from "../../known-failures.mjs";
|
|
4
|
-
|
|
5
|
-
export default class KnownFailuresRenderCommand extends Command {
|
|
6
|
-
static summary = "Render known failures markdown";
|
|
7
|
-
|
|
8
|
-
static enableJsonFlag = true;
|
|
9
|
-
|
|
10
|
-
static flags = makeKnownFailuresFlags();
|
|
11
|
-
|
|
12
|
-
async run() {
|
|
13
|
-
const { flags } = await this.parse(KnownFailuresRenderCommand);
|
|
14
|
-
await runKnownFailuresRenderCommand({
|
|
15
|
-
...flags,
|
|
16
|
-
});
|
|
17
|
-
return { ok: true };
|
|
18
|
-
}
|
|
19
|
-
}
|