@elench/testkit 0.1.80 → 0.1.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -56
- package/lib/cli/args.mjs +2 -14
- package/lib/cli/args.test.mjs +1 -17
- package/lib/cli/command-helpers.mjs +1 -20
- package/lib/cli/entrypoint.mjs +0 -4
- package/lib/cli/presentation/colors.mjs +1 -1
- package/lib/cli/presentation/failure-presentation.mjs +4 -4
- package/lib/cli/presentation/run-reporter.mjs +23 -9
- package/lib/cli/presentation/run-reporter.test.mjs +12 -6
- package/lib/cli/presentation/summary-box.test.mjs +4 -4
- package/lib/cli/viewer.mjs +18 -19
- package/lib/config/index.mjs +6 -6
- package/lib/config/runtime.mjs +8 -8
- package/lib/config-api/auth-fixtures.mjs +762 -0
- package/lib/config-api/index.d.ts +96 -112
- package/lib/config-api/index.mjs +22 -12
- package/lib/config-api/index.test.mjs +61 -222
- package/lib/index.d.ts +29 -9
- package/lib/package.test.mjs +4 -4
- package/lib/{known-failures → regressions}/github.mjs +36 -78
- package/lib/regressions/github.test.mjs +324 -0
- package/lib/regressions/index.d.ts +189 -0
- package/lib/{known-failures → regressions}/index.mjs +90 -93
- package/lib/{known-failures → regressions}/index.test.mjs +37 -48
- package/lib/runner/formatting.mjs +49 -34
- package/lib/runner/formatting.test.mjs +16 -15
- package/lib/runner/metadata.mjs +1 -1
- package/lib/runner/orchestrator.mjs +7 -9
- package/lib/runner/regressions.mjs +304 -0
- package/lib/runner/{triage.test.mjs → regressions.test.mjs} +50 -36
- package/lib/runner/reporting.mjs +2 -2
- package/lib/runner/reporting.test.mjs +2 -2
- package/lib/runner/run-finalization.mjs +18 -30
- package/lib/runner/template-steps.mjs +2 -2
- package/lib/runtime/index.d.ts +50 -33
- package/lib/runtime/index.mjs +0 -1
- package/lib/runtime-src/k6/http-suite-runtime.js +147 -0
- package/lib/runtime-src/k6/http.js +80 -41
- package/lib/runtime-src/k6/scenario-suite.js +13 -110
- package/lib/runtime-src/k6/suite.js +13 -107
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/package.json +8 -8
- package/lib/cli/commands/known-failures/render.mjs +0 -19
- package/lib/cli/commands/known-failures/validate.mjs +0 -20
- package/lib/cli/known-failures.mjs +0 -164
- package/lib/config-api/profiles.mjs +0 -640
- package/lib/known-failures/github.test.mjs +0 -512
- package/lib/known-failures/index.d.ts +0 -192
- package/lib/runner/triage.mjs +0 -221
- /package/lib/{known-failures → regressions}/github-cache.mjs +0 -0
- /package/lib/{known-failures → regressions}/github-transport.mjs +0 -0
package/lib/runner/triage.mjs
DELETED
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
buildKnownFailureFileIdentity,
|
|
3
|
-
findMatchingKnownFailureEntries,
|
|
4
|
-
loadKnownFailuresConfig,
|
|
5
|
-
} from "../known-failures/index.mjs";
|
|
6
|
-
|
|
7
|
-
export { loadKnownFailuresConfig };
|
|
8
|
-
|
|
9
|
-
export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures) {
|
|
10
|
-
if (!knownFailures) return { runArtifact, statusArtifact };
|
|
11
|
-
|
|
12
|
-
const runEntries = extractRunFileEntries(runArtifact);
|
|
13
|
-
const statusEntries = extractStatusFileEntries(statusArtifact);
|
|
14
|
-
const fileSummaries = new Map();
|
|
15
|
-
|
|
16
|
-
for (const entry of [...runEntries, ...statusEntries]) {
|
|
17
|
-
const key = buildKnownFailureFileIdentity(entry.service, entry.type, entry.path);
|
|
18
|
-
if (!fileSummaries.has(key)) {
|
|
19
|
-
fileSummaries.set(key, {
|
|
20
|
-
service: entry.service,
|
|
21
|
-
type: entry.type,
|
|
22
|
-
path: entry.path,
|
|
23
|
-
status: entry.status,
|
|
24
|
-
error: entry.error || null,
|
|
25
|
-
failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const matchesByFileKey = new Map();
|
|
31
|
-
const matchedByFailedEntryIds = new Set();
|
|
32
|
-
|
|
33
|
-
for (const fileSummary of fileSummaries.values()) {
|
|
34
|
-
const matches = findMatchingKnownFailureEntries(knownFailures, fileSummary);
|
|
35
|
-
if (matches.length === 0) continue;
|
|
36
|
-
|
|
37
|
-
const fileKey = buildKnownFailureFileIdentity(
|
|
38
|
-
fileSummary.service,
|
|
39
|
-
fileSummary.type,
|
|
40
|
-
fileSummary.path
|
|
41
|
-
);
|
|
42
|
-
matchesByFileKey.set(fileKey, matches.map((entry) => toArtifactTriageEntry(entry)));
|
|
43
|
-
if (fileSummary.status === "failed") {
|
|
44
|
-
for (const entry of matches) {
|
|
45
|
-
matchedByFailedEntryIds.add(entry.id);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
for (const entry of [...runEntries, ...statusEntries]) {
|
|
51
|
-
const fileKey = buildKnownFailureFileIdentity(entry.service, entry.type, entry.path);
|
|
52
|
-
const matches = matchesByFileKey.get(fileKey) || [];
|
|
53
|
-
if (matches.length === 0) {
|
|
54
|
-
if (entry.status === "failed") {
|
|
55
|
-
setEntryTriage(entry, {
|
|
56
|
-
status: "untriaged",
|
|
57
|
-
entries: [],
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
|
-
continue;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
setEntryTriage(entry, {
|
|
64
|
-
status: entry.status === "failed" ? "known_failure" : "known_issue_not_reproduced",
|
|
65
|
-
classifications: [...new Set(matches.map((match) => match.classification))].sort(),
|
|
66
|
-
entries: matches,
|
|
67
|
-
});
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const summaryTests = statusArtifact?.tests || runEntries;
|
|
71
|
-
const triageSummary = buildTriageSummary(
|
|
72
|
-
summaryTests,
|
|
73
|
-
knownFailures.entries,
|
|
74
|
-
matchedByFailedEntryIds
|
|
75
|
-
);
|
|
76
|
-
runArtifact.triageSummary = triageSummary;
|
|
77
|
-
if (statusArtifact) {
|
|
78
|
-
statusArtifact.triageSummary = triageSummary;
|
|
79
|
-
}
|
|
80
|
-
return { runArtifact, statusArtifact };
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export function applyKnownFailureIssueValidationToArtifacts(
|
|
84
|
-
runArtifact,
|
|
85
|
-
statusArtifact,
|
|
86
|
-
issueValidation
|
|
87
|
-
) {
|
|
88
|
-
if (!issueValidation) return { runArtifact, statusArtifact };
|
|
89
|
-
|
|
90
|
-
const validationById = new Map(
|
|
91
|
-
(issueValidation.entries || []).map((entry) => [entry.id, entry])
|
|
92
|
-
);
|
|
93
|
-
|
|
94
|
-
for (const entry of [...extractRunFileEntries(runArtifact), ...extractStatusFileEntries(statusArtifact)]) {
|
|
95
|
-
const triage = entry.target ? entry.target.triage : entry.triage;
|
|
96
|
-
if (!triage?.entries?.length) continue;
|
|
97
|
-
|
|
98
|
-
const matchedValidationEntries = triage.entries
|
|
99
|
-
.map((triageEntry) => validationById.get(triageEntry.id))
|
|
100
|
-
.filter(Boolean);
|
|
101
|
-
|
|
102
|
-
if (matchedValidationEntries.length === 0) continue;
|
|
103
|
-
|
|
104
|
-
const enrichedEntries = triage.entries.map((triageEntry) => {
|
|
105
|
-
const validationEntry = validationById.get(triageEntry.id);
|
|
106
|
-
if (!validationEntry) return triageEntry;
|
|
107
|
-
return {
|
|
108
|
-
...triageEntry,
|
|
109
|
-
github: validationEntry.github,
|
|
110
|
-
validationStatus: validationEntry.status,
|
|
111
|
-
findings: validationEntry.findings,
|
|
112
|
-
};
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
const nextTriage = {
|
|
116
|
-
...triage,
|
|
117
|
-
entries: enrichedEntries,
|
|
118
|
-
availability: summarizeIssueValidationAvailability(issueValidation, matchedValidationEntries),
|
|
119
|
-
};
|
|
120
|
-
setEntryTriage(entry, nextTriage);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
return { runArtifact, statusArtifact };
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function toArtifactTriageEntry(entry) {
|
|
127
|
-
return {
|
|
128
|
-
id: entry.id,
|
|
129
|
-
title: entry.title,
|
|
130
|
-
classification: entry.classification,
|
|
131
|
-
state: entry.state,
|
|
132
|
-
issue: entry.issue,
|
|
133
|
-
description: entry.description,
|
|
134
|
-
whyFailing: entry.whyFailing,
|
|
135
|
-
lastReviewedAt: entry.lastReviewedAt,
|
|
136
|
-
};
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function buildTriageSummary(tests, entries, matchedEntryIds) {
|
|
140
|
-
const failedTests = tests.filter((test) => test.status === "failed");
|
|
141
|
-
const knownFailedTests = failedTests.filter((test) => test.triage?.status === "known_failure");
|
|
142
|
-
const byClassification = {};
|
|
143
|
-
|
|
144
|
-
for (const test of knownFailedTests) {
|
|
145
|
-
for (const classification of test.triage?.classifications || []) {
|
|
146
|
-
byClassification[classification] = (byClassification[classification] || 0) + 1;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
return {
|
|
151
|
-
failed: {
|
|
152
|
-
total: failedTests.length,
|
|
153
|
-
known: knownFailedTests.length,
|
|
154
|
-
untriaged: failedTests.length - knownFailedTests.length,
|
|
155
|
-
byClassification,
|
|
156
|
-
},
|
|
157
|
-
entries: {
|
|
158
|
-
total: entries.length,
|
|
159
|
-
matchedByFailedTests: matchedEntryIds.size,
|
|
160
|
-
unmatched: entries.length - matchedEntryIds.size,
|
|
161
|
-
},
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
function extractRunFileEntries(runArtifact) {
|
|
166
|
-
const entries = [];
|
|
167
|
-
|
|
168
|
-
for (const service of runArtifact.services || []) {
|
|
169
|
-
for (const suite of service.suites || []) {
|
|
170
|
-
for (const file of suite.files || []) {
|
|
171
|
-
entries.push({
|
|
172
|
-
target: file,
|
|
173
|
-
service: service.name,
|
|
174
|
-
type: suite.type,
|
|
175
|
-
path: file.path,
|
|
176
|
-
status: file.status,
|
|
177
|
-
error: file.error || null,
|
|
178
|
-
failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
return entries;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function extractStatusFileEntries(statusArtifact) {
|
|
188
|
-
return statusArtifact?.tests || [];
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
function setEntryTriage(entry, triage) {
|
|
192
|
-
if (entry?.target) {
|
|
193
|
-
entry.target.triage = triage;
|
|
194
|
-
return;
|
|
195
|
-
}
|
|
196
|
-
entry.triage = triage;
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function summarizeIssueValidationAvailability(issueValidation, entries) {
|
|
200
|
-
if (entries.some((entry) => entry.github?.cached)) {
|
|
201
|
-
return {
|
|
202
|
-
mode: "cache",
|
|
203
|
-
reason: issueValidation.availability?.usedCachedFallback
|
|
204
|
-
? "used stale cache"
|
|
205
|
-
: "used cached issue metadata",
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
if (entries.some((entry) => entry.status === "validation_unavailable")) {
|
|
209
|
-
const globalReason = (issueValidation.findings || []).find(
|
|
210
|
-
(finding) => finding.code === "validation_unavailable"
|
|
211
|
-
);
|
|
212
|
-
return {
|
|
213
|
-
mode: "offline",
|
|
214
|
-
reason: globalReason?.message || "validation unavailable",
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
return {
|
|
218
|
-
mode: "live",
|
|
219
|
-
reason: null,
|
|
220
|
-
};
|
|
221
|
-
}
|
|
File without changes
|
|
File without changes
|