@elench/testkit 0.1.149 → 0.1.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -9
- package/lib/cli/assistant/view-model.mjs +1 -1
- package/lib/cli/components/blocks/run-tree.mjs +1 -1
- package/lib/cli/renderers/run/events.mjs +4 -3
- package/lib/cli/renderers/run/failure.mjs +2 -2
- package/lib/cli/renderers/run/inline-detail.mjs +2 -2
- package/lib/cli/renderers/run/interactive.mjs +2 -2
- package/lib/cli/renderers/run/text-reporter.mjs +9 -9
- package/lib/cli/state/run/model.mjs +7 -7
- package/lib/cli/state/run/state.mjs +3 -3
- package/lib/cli/terminal/colors.mjs +1 -1
- package/lib/config/database-materialization.mjs +25 -0
- package/lib/config/database.mjs +30 -0
- package/lib/config/index.mjs +47 -1
- package/lib/config/runtime.mjs +130 -0
- package/lib/config-api/index.d.ts +28 -0
- package/lib/config-api/index.mjs +6 -0
- package/lib/database/cleanup.mjs +76 -1
- package/lib/database/constants.mjs +3 -0
- package/lib/database/index.mjs +6 -0
- package/lib/database/local-postgres.mjs +123 -4
- package/lib/database/naming.mjs +7 -0
- package/lib/database/resource-postgres.mjs +13 -0
- package/lib/database/state-files.mjs +17 -0
- package/lib/docker-compat/matrix.mjs +5 -3
- package/lib/kiln/client.mjs +8 -0
- package/lib/local/kiln-driver.mjs +96 -68
- package/lib/ownership/docker.mjs +67 -1
- package/lib/regressions/github-transport.mjs +178 -4
- package/lib/regressions/github.mjs +52 -16
- package/lib/regressions/index.d.ts +58 -29
- package/lib/regressions/index.mjs +171 -58
- package/lib/regressions/workflow.mjs +266 -0
- package/lib/results/artifacts.mjs +8 -7
- package/lib/runner/formatting.mjs +17 -16
- package/lib/runner/orchestrator.mjs +6 -5
- package/lib/runner/planning.mjs +40 -0
- package/lib/runner/regressions.mjs +183 -33
- package/lib/runner/reporting.mjs +1 -1
- package/lib/runner/run-finalization.mjs +34 -4
- package/lib/runner/runtime-manager.mjs +91 -10
- package/lib/runner/scheduler/index.mjs +30 -1
- package/lib/runtime/index.d.ts +5 -5
- package/lib/runtime-src/k6/http.js +11 -11
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/node_modules/es-toolkit/CHANGELOG.md +801 -0
- package/node_modules/es-toolkit/src/compat/_internal/Equals.d.ts +1 -0
- package/node_modules/es-toolkit/src/compat/_internal/IsWritable.d.ts +3 -0
- package/node_modules/es-toolkit/src/compat/_internal/MutableList.d.ts +4 -0
- package/node_modules/es-toolkit/src/compat/_internal/RejectReadonly.d.ts +4 -0
- package/node_modules/esprima/ChangeLog +235 -0
- package/package.json +6 -5
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts +0 -188
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js +0 -293
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/package.json +0 -25
|
@@ -3,117 +3,152 @@ import path from "path";
|
|
|
3
3
|
|
|
4
4
|
const CLASSIFICATIONS = new Set([
|
|
5
5
|
"expected_failure",
|
|
6
|
+
"flaky",
|
|
7
|
+
"harness_bug",
|
|
6
8
|
"infra",
|
|
7
9
|
"product_bug",
|
|
8
10
|
"stale_test",
|
|
9
11
|
"test_bug",
|
|
10
12
|
]);
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
const CASE_STATES = new Set([
|
|
15
|
+
"active",
|
|
16
|
+
"coverage_missing",
|
|
17
|
+
"fixed_pending_verification",
|
|
18
|
+
"fix_claimed",
|
|
19
|
+
"resolved",
|
|
20
|
+
"untriaged",
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
export function loadRegressionCaseStoreConfig(productDir, config) {
|
|
13
24
|
const relativePath = config?.file;
|
|
14
25
|
if (!relativePath) return null;
|
|
15
26
|
|
|
16
27
|
const absolutePath = path.resolve(productDir, relativePath);
|
|
17
28
|
if (!fs.existsSync(absolutePath)) {
|
|
18
|
-
throw new Error(`Regression
|
|
29
|
+
throw new Error(`Regression case store not found: ${relativePath}`);
|
|
19
30
|
}
|
|
20
31
|
|
|
21
|
-
|
|
32
|
+
const document = loadRegressionCaseStoreDocument(absolutePath, relativePath);
|
|
33
|
+
Object.defineProperty(document, "__filePath", {
|
|
34
|
+
value: absolutePath,
|
|
35
|
+
enumerable: false,
|
|
36
|
+
configurable: false,
|
|
37
|
+
});
|
|
38
|
+
return document;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function writeRegressionCaseStoreDocument(filePath, document) {
|
|
42
|
+
fs.writeFileSync(filePath, `${JSON.stringify(serializeRegressionCaseStore(document), null, 2)}\n`);
|
|
22
43
|
}
|
|
23
44
|
|
|
24
|
-
export function
|
|
45
|
+
export function serializeRegressionCaseStore(document) {
|
|
46
|
+
const normalized = normalizeRegressionCaseStoreDocument(document);
|
|
47
|
+
return {
|
|
48
|
+
schemaVersion: 2,
|
|
49
|
+
issueRepo: normalized.issueRepo,
|
|
50
|
+
cases: normalized.cases,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function loadRegressionCaseStoreDocument(filePath, relativePath = filePath) {
|
|
25
55
|
let parsed;
|
|
26
56
|
try {
|
|
27
57
|
parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
28
58
|
} catch (error) {
|
|
29
59
|
throw new Error(
|
|
30
|
-
`Could not parse regression
|
|
60
|
+
`Could not parse regression case store ${relativePath}: ${formatErrorMessage(error)}`
|
|
31
61
|
);
|
|
32
62
|
}
|
|
33
63
|
|
|
34
|
-
return
|
|
64
|
+
return normalizeRegressionCaseStoreDocument(parsed, relativePath);
|
|
35
65
|
}
|
|
36
66
|
|
|
37
|
-
export function
|
|
67
|
+
export function normalizeRegressionCaseStoreDocument(document, relativePath = "regression case store") {
|
|
38
68
|
if (!document || typeof document !== "object") {
|
|
39
69
|
throw new Error(`${relativePath} must contain a JSON object`);
|
|
40
70
|
}
|
|
41
|
-
if (document.schemaVersion !==
|
|
42
|
-
throw new Error(`${relativePath} schemaVersion must be
|
|
71
|
+
if (document.schemaVersion !== 2) {
|
|
72
|
+
throw new Error(`${relativePath} schemaVersion must be 2`);
|
|
43
73
|
}
|
|
44
|
-
if (!Array.isArray(document.
|
|
45
|
-
throw new Error(`${relativePath}
|
|
74
|
+
if (!Array.isArray(document.cases)) {
|
|
75
|
+
throw new Error(`${relativePath} cases must be an array`);
|
|
46
76
|
}
|
|
47
77
|
|
|
48
78
|
const ids = new Set();
|
|
49
|
-
const
|
|
50
|
-
const normalized =
|
|
79
|
+
const cases = document.cases.map((entry, index) => {
|
|
80
|
+
const normalized = normalizeRegressionCase(entry, `${relativePath} cases[${index}]`);
|
|
51
81
|
if (ids.has(normalized.id)) {
|
|
52
|
-
throw new Error(`${relativePath} has duplicate
|
|
82
|
+
throw new Error(`${relativePath} has duplicate case id "${normalized.id}"`);
|
|
53
83
|
}
|
|
54
84
|
ids.add(normalized.id);
|
|
55
85
|
return normalized;
|
|
56
86
|
});
|
|
57
87
|
|
|
58
88
|
return {
|
|
59
|
-
schemaVersion:
|
|
89
|
+
schemaVersion: 2,
|
|
60
90
|
issueRepo: normalizeOptionalString(document.issueRepo),
|
|
61
|
-
|
|
91
|
+
cases,
|
|
62
92
|
};
|
|
63
93
|
}
|
|
64
94
|
|
|
65
|
-
export function
|
|
95
|
+
export function validateRegressionCaseStoreDocument(
|
|
66
96
|
document,
|
|
67
97
|
{ productDir, statusArtifactPath, statusArtifact } = {}
|
|
68
98
|
) {
|
|
99
|
+
const normalizedDocument = normalizeRegressionCaseStoreDocument(document);
|
|
69
100
|
const errors = [];
|
|
70
101
|
const warnings = [];
|
|
71
102
|
const fingerprintKeys = new Set();
|
|
72
103
|
let fingerprintCount = 0;
|
|
73
104
|
|
|
74
|
-
for (const
|
|
75
|
-
for (const fingerprint of
|
|
105
|
+
for (const regressionCase of normalizedDocument.cases) {
|
|
106
|
+
for (const fingerprint of regressionCase.fingerprints) {
|
|
76
107
|
fingerprintCount += 1;
|
|
77
108
|
if (productDir) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
109
|
+
if (fingerprint.path) {
|
|
110
|
+
const absolutePath = path.join(productDir, fingerprint.path);
|
|
111
|
+
if (!fs.existsSync(absolutePath)) {
|
|
112
|
+
errors.push(`Missing matched file: ${fingerprint.path} (${regressionCase.id})`);
|
|
113
|
+
}
|
|
114
|
+
} else if (!fingerprint.pathGlob) {
|
|
115
|
+
errors.push(`Missing matched file: ${fingerprint.path} (${regressionCase.id})`);
|
|
81
116
|
}
|
|
82
117
|
}
|
|
83
118
|
|
|
84
119
|
const fingerprintKey = [
|
|
85
|
-
|
|
120
|
+
regressionCase.id,
|
|
86
121
|
fingerprint.service || "",
|
|
87
122
|
fingerprint.type || "",
|
|
88
|
-
fingerprint.path,
|
|
123
|
+
fingerprint.path || "",
|
|
124
|
+
fingerprint.pathGlob || "",
|
|
89
125
|
fingerprint.failureKey || "",
|
|
90
126
|
fingerprint.errorIncludes || "",
|
|
91
127
|
].join("::");
|
|
92
128
|
if (fingerprintKeys.has(fingerprintKey)) {
|
|
93
|
-
errors.push(`Duplicate fingerprint selector in ${
|
|
129
|
+
errors.push(`Duplicate fingerprint selector in ${regressionCase.id}: ${fingerprint.path}`);
|
|
94
130
|
}
|
|
95
131
|
fingerprintKeys.add(fingerprintKey);
|
|
96
132
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
!fingerprint.path.endsWith(".testkit.ts")
|
|
100
|
-
) {
|
|
133
|
+
const displayPath = fingerprint.path || fingerprint.pathGlob;
|
|
134
|
+
if (!displayPath.includes("__testkit__") || !displayPath.endsWith(".testkit.ts")) {
|
|
101
135
|
warnings.push(
|
|
102
|
-
`Fingerprint path does not look like a testkit file: ${
|
|
136
|
+
`Fingerprint path does not look like a testkit file: ${displayPath} (${regressionCase.id})`
|
|
103
137
|
);
|
|
104
138
|
}
|
|
105
139
|
}
|
|
106
140
|
|
|
107
|
-
if (
|
|
141
|
+
if (
|
|
142
|
+
normalizedDocument.issueRepo &&
|
|
143
|
+
regressionCase.issue &&
|
|
144
|
+
regressionCase.issue.repo !== normalizedDocument.issueRepo
|
|
145
|
+
) {
|
|
108
146
|
warnings.push(
|
|
109
|
-
`
|
|
147
|
+
`Case ${regressionCase.id} uses issue repo ${regressionCase.issue.repo} instead of document issueRepo ${normalizedDocument.issueRepo}`
|
|
110
148
|
);
|
|
111
149
|
}
|
|
112
150
|
}
|
|
113
151
|
|
|
114
|
-
let failedTests = 0;
|
|
115
|
-
let diagnosedFailedTests = 0;
|
|
116
|
-
let newFailedTests = 0;
|
|
117
152
|
const parsedStatusArtifact = statusArtifact
|
|
118
153
|
? parseStatusArtifact(statusArtifact)
|
|
119
154
|
: statusArtifactPath && fs.existsSync(statusArtifactPath)
|
|
@@ -121,13 +156,14 @@ export function validateRegressionCatalogDocument(
|
|
|
121
156
|
: { tests: [] };
|
|
122
157
|
|
|
123
158
|
const failed = parsedStatusArtifact.tests.filter((test) => test.status === "failed");
|
|
124
|
-
|
|
159
|
+
let diagnosedFailedTests = 0;
|
|
160
|
+
let newFailedTests = 0;
|
|
125
161
|
for (const test of failed) {
|
|
126
|
-
if (
|
|
162
|
+
if (findMatchingRegressionCases(normalizedDocument, test).length > 0) {
|
|
127
163
|
diagnosedFailedTests += 1;
|
|
128
164
|
} else {
|
|
129
165
|
newFailedTests += 1;
|
|
130
|
-
warnings.push(`New failing test not yet in regression
|
|
166
|
+
warnings.push(`New failing test not yet in regression case store: ${test.path}`);
|
|
131
167
|
}
|
|
132
168
|
}
|
|
133
169
|
|
|
@@ -135,30 +171,37 @@ export function validateRegressionCatalogDocument(
|
|
|
135
171
|
errors,
|
|
136
172
|
warnings,
|
|
137
173
|
stats: {
|
|
138
|
-
|
|
174
|
+
cases: normalizedDocument.cases.length,
|
|
139
175
|
fingerprints: fingerprintCount,
|
|
140
|
-
failedTests,
|
|
176
|
+
failedTests: failed.length,
|
|
141
177
|
diagnosedFailedTests,
|
|
142
178
|
newFailedTests,
|
|
143
179
|
},
|
|
144
180
|
};
|
|
145
181
|
}
|
|
146
182
|
|
|
147
|
-
export function
|
|
183
|
+
export function renderRegressionCaseStoreMarkdown(document) {
|
|
184
|
+
const normalizedDocument = normalizeRegressionCaseStoreDocument(document);
|
|
185
|
+
const issueRefs = new Set(
|
|
186
|
+
normalizedDocument.cases
|
|
187
|
+
.map((entry) => entry.issue ? `${entry.issue.repo}#${entry.issue.number}` : null)
|
|
188
|
+
.filter(Boolean)
|
|
189
|
+
);
|
|
148
190
|
const lines = [
|
|
149
|
-
"# Regression
|
|
191
|
+
"# Regression Cases",
|
|
150
192
|
"",
|
|
151
193
|
"Canonical source: `testkit.regressions.json`",
|
|
152
194
|
"",
|
|
153
|
-
`Tracked
|
|
154
|
-
`Tracked issue references: ${
|
|
195
|
+
`Tracked cases: ${normalizedDocument.cases.length}`,
|
|
196
|
+
`Tracked issue references: ${issueRefs.size}`,
|
|
155
197
|
"",
|
|
156
198
|
];
|
|
157
199
|
|
|
158
|
-
|
|
200
|
+
normalizedDocument.cases.forEach((entry, index) => {
|
|
159
201
|
lines.push(
|
|
160
|
-
`## ${index + 1}. ${entry.summary}
|
|
202
|
+
`## ${index + 1}. ${entry.summary}${entry.issue ? ` - #${entry.issue.number}` : ""}`,
|
|
161
203
|
"",
|
|
204
|
+
`- State: \`${entry.state}\``,
|
|
162
205
|
`- Classification: \`${entry.classification}\``,
|
|
163
206
|
`- Cause: ${entry.cause}`,
|
|
164
207
|
`- Last reviewed: ${entry.lastReviewedAt}`,
|
|
@@ -167,8 +210,8 @@ export function renderRegressionCatalogMarkdown(document) {
|
|
|
167
210
|
|
|
168
211
|
for (const fingerprint of entry.fingerprints) {
|
|
169
212
|
lines.push(
|
|
170
|
-
` - \`${fingerprint.path}\`${fingerprint.failureKey ? `
|
|
171
|
-
fingerprint.errorIncludes ? `
|
|
213
|
+
` - \`${fingerprint.path || fingerprint.pathGlob}\`${fingerprint.failureKey ? ` - \`${fingerprint.failureKey}\`` : ""}${
|
|
214
|
+
fingerprint.errorIncludes ? ` - error contains \`${fingerprint.errorIncludes}\`` : ""
|
|
172
215
|
}`
|
|
173
216
|
);
|
|
174
217
|
}
|
|
@@ -179,11 +222,13 @@ export function renderRegressionCatalogMarkdown(document) {
|
|
|
179
222
|
return `${lines.join("\n").trimEnd()}\n`;
|
|
180
223
|
}
|
|
181
224
|
|
|
182
|
-
export function
|
|
183
|
-
return document.
|
|
225
|
+
export function findMatchingRegressionCases(document, fileSummary) {
|
|
226
|
+
return (document?.cases || []).filter((entry) =>
|
|
227
|
+
matchesRegressionCase(entry, fileSummary)
|
|
228
|
+
);
|
|
184
229
|
}
|
|
185
230
|
|
|
186
|
-
export function
|
|
231
|
+
export function matchesRegressionCase(entry, fileSummary) {
|
|
187
232
|
return entry.fingerprints.some((fingerprint) =>
|
|
188
233
|
matchesRegressionFingerprint(fingerprint, fileSummary)
|
|
189
234
|
);
|
|
@@ -192,7 +237,8 @@ export function matchesRegressionEntry(entry, fileSummary) {
|
|
|
192
237
|
export function matchesRegressionFingerprint(fingerprint, fileSummary) {
|
|
193
238
|
if (fingerprint.service && fingerprint.service !== fileSummary.service) return false;
|
|
194
239
|
if (fingerprint.type && fingerprint.type !== fileSummary.type) return false;
|
|
195
|
-
if (fingerprint.path !== fileSummary.path) return false;
|
|
240
|
+
if (fingerprint.path && fingerprint.path !== fileSummary.path) return false;
|
|
241
|
+
if (fingerprint.pathGlob && !globMatches(fingerprint.pathGlob, fileSummary.path)) return false;
|
|
196
242
|
if (fingerprint.failureKey) {
|
|
197
243
|
const failureKeys = Array.isArray(fileSummary.failureDetails)
|
|
198
244
|
? fileSummary.failureDetails.flatMap((detail) => [detail?.key, detail?.title].filter(Boolean))
|
|
@@ -209,17 +255,24 @@ export function buildRegressionFileIdentity(service, type, filePath) {
|
|
|
209
255
|
return `${service}::${type}::${filePath}`;
|
|
210
256
|
}
|
|
211
257
|
|
|
212
|
-
function
|
|
258
|
+
export function issueRefToString(issue) {
|
|
259
|
+
if (!issue) return null;
|
|
260
|
+
return `${issue.repo}#${issue.number}`;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function normalizeRegressionCase(entry, label) {
|
|
213
264
|
if (!entry || typeof entry !== "object") {
|
|
214
265
|
throw new Error(`${label} must be an object`);
|
|
215
266
|
}
|
|
216
267
|
|
|
217
268
|
const id = requireNonEmptyString(entry.id, `${label}.id`);
|
|
269
|
+
const state = requireEnumValue(entry.state || "active", CASE_STATES, `${label}.state`);
|
|
218
270
|
const classification = requireEnumValue(
|
|
219
271
|
entry.classification,
|
|
220
272
|
CLASSIFICATIONS,
|
|
221
273
|
`${label}.classification`
|
|
222
274
|
);
|
|
275
|
+
const owner = normalizeOptionalString(entry.owner);
|
|
223
276
|
const summary = requireNonEmptyString(entry.summary, `${label}.summary`);
|
|
224
277
|
const cause = requireNonEmptyString(entry.cause, `${label}.cause`);
|
|
225
278
|
const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
|
|
@@ -229,14 +282,18 @@ function normalizeRegressionEntry(entry, label) {
|
|
|
229
282
|
|
|
230
283
|
return {
|
|
231
284
|
id,
|
|
285
|
+
state,
|
|
232
286
|
classification,
|
|
233
|
-
|
|
287
|
+
...(owner ? { owner } : {}),
|
|
288
|
+
issue: entry.issue == null ? null : normalizeRegressionIssue(entry.issue, `${label}.issue`),
|
|
234
289
|
summary,
|
|
235
290
|
cause,
|
|
236
291
|
lastReviewedAt,
|
|
237
292
|
fingerprints: entry.fingerprints.map((fingerprint, index) =>
|
|
238
293
|
normalizeRegressionFingerprint(fingerprint, `${label}.fingerprints[${index}]`)
|
|
239
294
|
),
|
|
295
|
+
lifecycle: normalizeLifecycle(entry.lifecycle),
|
|
296
|
+
coverage: normalizeCoverage(entry.coverage),
|
|
240
297
|
};
|
|
241
298
|
}
|
|
242
299
|
|
|
@@ -254,15 +311,39 @@ function normalizeRegressionIssue(issue, label) {
|
|
|
254
311
|
return { repo, number };
|
|
255
312
|
}
|
|
256
313
|
|
|
314
|
+
function normalizeLifecycle(value) {
|
|
315
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
316
|
+
return {
|
|
317
|
+
...(normalizeOptionalString(value.firstSeenAt) ? { firstSeenAt: normalizeOptionalString(value.firstSeenAt) } : {}),
|
|
318
|
+
...(normalizeOptionalString(value.lastSeenAt) ? { lastSeenAt: normalizeOptionalString(value.lastSeenAt) } : {}),
|
|
319
|
+
...(normalizeOptionalString(value.lastVerifiedAt) ? { lastVerifiedAt: normalizeOptionalString(value.lastVerifiedAt) } : {}),
|
|
320
|
+
...(normalizeOptionalString(value.resolvedAt) ? { resolvedAt: normalizeOptionalString(value.resolvedAt) } : {}),
|
|
321
|
+
...(value.cleanRunCount != null ? { cleanRunCount: Math.max(0, Number(value.cleanRunCount) || 0) } : {}),
|
|
322
|
+
...(value.fixClaim && typeof value.fixClaim === "object" && !Array.isArray(value.fixClaim)
|
|
323
|
+
? { fixClaim: { ...value.fixClaim } }
|
|
324
|
+
: {}),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function normalizeCoverage(value) {
|
|
329
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
330
|
+
return {
|
|
331
|
+
...(typeof value.required === "boolean" ? { required: value.required } : {}),
|
|
332
|
+
...(normalizeOptionalString(value.status) ? { status: normalizeOptionalString(value.status) } : {}),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
|
|
257
336
|
function normalizeRegressionFingerprint(fingerprint, label) {
|
|
258
337
|
if (!fingerprint || typeof fingerprint !== "object") {
|
|
259
338
|
throw new Error(`${label} must be an object`);
|
|
260
339
|
}
|
|
261
340
|
|
|
262
|
-
const pathValue =
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
341
|
+
const pathValue = normalizeOptionalString(fingerprint.path);
|
|
342
|
+
const pathGlob = normalizeOptionalString(fingerprint.pathGlob);
|
|
343
|
+
if ((pathValue ? 1 : 0) + (pathGlob ? 1 : 0) !== 1) {
|
|
344
|
+
throw new Error(`${label} must specify exactly one of path or pathGlob`);
|
|
345
|
+
}
|
|
346
|
+
const normalized = pathValue ? { path: pathValue } : { pathGlob };
|
|
266
347
|
|
|
267
348
|
const service = normalizeOptionalString(fingerprint.service);
|
|
268
349
|
if (service) normalized.service = service;
|
|
@@ -276,9 +357,41 @@ function normalizeRegressionFingerprint(fingerprint, label) {
|
|
|
276
357
|
const errorIncludes = normalizeOptionalString(fingerprint.errorIncludes);
|
|
277
358
|
if (errorIncludes) normalized.errorIncludes = errorIncludes;
|
|
278
359
|
|
|
360
|
+
if (pathGlob && (!normalized.service || !normalized.type || (!normalized.failureKey && !normalized.errorIncludes))) {
|
|
361
|
+
throw new Error(`${label}.pathGlob requires service, type, and failureKey or errorIncludes`);
|
|
362
|
+
}
|
|
363
|
+
|
|
279
364
|
return normalized;
|
|
280
365
|
}
|
|
281
366
|
|
|
367
|
+
function globMatches(pattern, value) {
|
|
368
|
+
return globToRegExp(pattern).test(String(value || ""));
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function globToRegExp(pattern) {
|
|
372
|
+
let source = "^";
|
|
373
|
+
for (let index = 0; index < pattern.length; index += 1) {
|
|
374
|
+
const char = pattern[index];
|
|
375
|
+
const next = pattern[index + 1];
|
|
376
|
+
if (char === "*" && next === "*") {
|
|
377
|
+
source += ".*";
|
|
378
|
+
index += 1;
|
|
379
|
+
} else if (char === "*") {
|
|
380
|
+
source += "[^/]*";
|
|
381
|
+
} else if (char === "?") {
|
|
382
|
+
source += "[^/]";
|
|
383
|
+
} else {
|
|
384
|
+
source += escapeRegExp(char);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
source += "$";
|
|
388
|
+
return new RegExp(source);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function escapeRegExp(value) {
|
|
392
|
+
return String(value).replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
|
|
393
|
+
}
|
|
394
|
+
|
|
282
395
|
function parseStatusArtifact(value) {
|
|
283
396
|
if (!value || typeof value !== "object") {
|
|
284
397
|
return { tests: [] };
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { execFileSync } from "child_process";
|
|
2
|
+
import { createDefaultGitHubIssueTransport } from "./github-transport.mjs";
|
|
3
|
+
|
|
4
|
+
const STRONG_FIX_RE = /\b(?:fixe[sd]?|close[sd]?|resolve[sd]?)\s+((?:[\w.-]+\/[\w.-]+)?#\d+)/giu;
|
|
5
|
+
const WEAK_REF_RE = /\b(?:refs?|references?|related to)\s+((?:[\w.-]+\/[\w.-]+)?#\d+)/giu;
|
|
6
|
+
|
|
7
|
+
export function parseIssueReferences(text, defaultRepo = null) {
|
|
8
|
+
const refs = [];
|
|
9
|
+
collectRefs(refs, text, STRONG_FIX_RE, "fix");
|
|
10
|
+
collectRefs(refs, text, WEAK_REF_RE, "reference");
|
|
11
|
+
const seen = new Set();
|
|
12
|
+
return refs
|
|
13
|
+
.map((ref) => normalizeIssueRef(ref, defaultRepo))
|
|
14
|
+
.filter(Boolean)
|
|
15
|
+
.filter((ref) => {
|
|
16
|
+
const key = `${ref.kind}:${ref.repo}#${ref.number}`;
|
|
17
|
+
if (seen.has(key)) return false;
|
|
18
|
+
seen.add(key);
|
|
19
|
+
return true;
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function collectLocalFixClaims(productDir, gitMetadata = {}) {
|
|
24
|
+
const defaultRepo = gitMetadata?.repoSlug || null;
|
|
25
|
+
const messages = [];
|
|
26
|
+
try {
|
|
27
|
+
messages.push(execFileSync("git", ["log", "-20", "--pretty=%B"], {
|
|
28
|
+
cwd: productDir,
|
|
29
|
+
encoding: "utf8",
|
|
30
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
31
|
+
}));
|
|
32
|
+
} catch {
|
|
33
|
+
return [];
|
|
34
|
+
}
|
|
35
|
+
return parseIssueReferences(messages.join("\n"), defaultRepo).filter((ref) => ref.kind === "fix");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function executeRegressionWorkflowActions({
|
|
39
|
+
productDir,
|
|
40
|
+
caseStore,
|
|
41
|
+
report,
|
|
42
|
+
workflowConfig,
|
|
43
|
+
gitMetadata,
|
|
44
|
+
env = process.env,
|
|
45
|
+
}) {
|
|
46
|
+
if (!caseStore || !report || !workflowConfig || workflowConfig.mode === "off") {
|
|
47
|
+
return { changed: false, actions: [] };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const actions = [];
|
|
51
|
+
let changed = false;
|
|
52
|
+
const transport = await maybeCreateGitHubTransport(workflowConfig, env);
|
|
53
|
+
|
|
54
|
+
if (workflowConfig?.github?.read !== false && transport?.fetchOpenBugIssues && caseStore.issueRepo) {
|
|
55
|
+
try {
|
|
56
|
+
const openBugIssues = await transport.fetchOpenBugIssues(caseStore.issueRepo, {
|
|
57
|
+
labels: workflowConfig.github?.bugLabels || ["bug"],
|
|
58
|
+
});
|
|
59
|
+
const coverage = await reconcileBugIssueCoverage({
|
|
60
|
+
caseStore,
|
|
61
|
+
issuesByNumber: openBugIssues,
|
|
62
|
+
issueRepo: caseStore.issueRepo,
|
|
63
|
+
});
|
|
64
|
+
if (coverage.changed) changed = true;
|
|
65
|
+
actions.push(...coverage.actions);
|
|
66
|
+
} catch (error) {
|
|
67
|
+
actions.push({
|
|
68
|
+
type: "sync_bug_coverage",
|
|
69
|
+
reason: "github_bug_issue_fetch_failed",
|
|
70
|
+
status: "suggested",
|
|
71
|
+
message: error instanceof Error ? error.message : String(error),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const fixClaims = collectLocalFixClaims(productDir, gitMetadata);
|
|
77
|
+
const cases = caseStore.cases || [];
|
|
78
|
+
const issueToCase = new Map(
|
|
79
|
+
cases
|
|
80
|
+
.filter((entry) => entry.issue)
|
|
81
|
+
.map((entry) => [`${entry.issue.repo}#${entry.issue.number}`, entry])
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
for (const claim of fixClaims) {
|
|
85
|
+
const regressionCase = issueToCase.get(`${claim.repo}#${claim.number}`);
|
|
86
|
+
if (!regressionCase) continue;
|
|
87
|
+
const previousState = JSON.stringify({
|
|
88
|
+
state: regressionCase.state,
|
|
89
|
+
lifecycle: regressionCase.lifecycle || {},
|
|
90
|
+
});
|
|
91
|
+
regressionCase.lifecycle = {
|
|
92
|
+
...(regressionCase.lifecycle || {}),
|
|
93
|
+
fixClaim: {
|
|
94
|
+
repo: claim.repo,
|
|
95
|
+
number: claim.number,
|
|
96
|
+
source: "git",
|
|
97
|
+
observedAt: new Date().toISOString(),
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
if (regressionCase.state === "active") regressionCase.state = "fix_claimed";
|
|
101
|
+
changed = changed || previousState !== JSON.stringify({
|
|
102
|
+
state: regressionCase.state,
|
|
103
|
+
lifecycle: regressionCase.lifecycle || {},
|
|
104
|
+
});
|
|
105
|
+
actions.push({
|
|
106
|
+
type: "record_fix_claim",
|
|
107
|
+
caseId: regressionCase.id,
|
|
108
|
+
issue: regressionCase.issue,
|
|
109
|
+
reason: "commit_message_fix_reference",
|
|
110
|
+
status: "applied",
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (workflowConfig?.github?.write && transport) {
|
|
115
|
+
const writeResult = await executeGitHubWriteActions({ transport, caseStore, report, actions, workflowConfig });
|
|
116
|
+
if (writeResult.changed) changed = true;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (actions.length > 0) {
|
|
120
|
+
report.workflow = report.workflow || { actions: [], summary: { applied: 0, suggested: 0, byType: {} } };
|
|
121
|
+
report.workflow.actions = [...(report.workflow.actions || []), ...actions];
|
|
122
|
+
report.workflow.summary = summarizeWorkflowActions(report.workflow.actions);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return { changed: changed || actions.some((action) => action.status === "applied"), actions };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export async function reconcileBugIssueCoverage({ caseStore, issuesByNumber, issueRepo }) {
|
|
129
|
+
const actions = [];
|
|
130
|
+
if (!caseStore || !issuesByNumber || issuesByNumber.size === 0) return { changed: false, actions };
|
|
131
|
+
const cases = caseStore.cases || [];
|
|
132
|
+
const coveredIssueNumbers = new Set(cases.filter((entry) => entry.issue).map((entry) => entry.issue.number));
|
|
133
|
+
|
|
134
|
+
for (const [number, issue] of issuesByNumber.entries()) {
|
|
135
|
+
if (coveredIssueNumbers.has(number)) continue;
|
|
136
|
+
const id = `github-bug-${number}`;
|
|
137
|
+
if (cases.some((entry) => entry.id === id)) continue;
|
|
138
|
+
cases.push({
|
|
139
|
+
id,
|
|
140
|
+
state: "coverage_missing",
|
|
141
|
+
classification: "product_bug",
|
|
142
|
+
issue: { repo: issue.repo || issueRepo, number },
|
|
143
|
+
summary: issue.title || `Bug #${number} lacks regression coverage`,
|
|
144
|
+
cause: "Open GitHub bug issue has no linked Testkit regression case.",
|
|
145
|
+
lastReviewedAt: new Date().toISOString().slice(0, 10),
|
|
146
|
+
fingerprints: [{ pathGlob: "**/__testkit__/*.testkit.ts", service: "unknown", type: "int", errorIncludes: id }],
|
|
147
|
+
lifecycle: {},
|
|
148
|
+
coverage: { required: true, status: "coverage_missing" },
|
|
149
|
+
});
|
|
150
|
+
actions.push({
|
|
151
|
+
type: "record_missing_bug_coverage",
|
|
152
|
+
caseId: id,
|
|
153
|
+
issue: { repo: issue.repo || issueRepo, number },
|
|
154
|
+
reason: "open_bug_issue_without_case",
|
|
155
|
+
status: "applied",
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return { changed: actions.length > 0, actions };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function executeGitHubWriteActions({ transport, caseStore, report, actions, workflowConfig }) {
|
|
163
|
+
const cases = caseStore.cases || [];
|
|
164
|
+
const caseById = new Map(cases.map((entry) => [entry.id, entry]));
|
|
165
|
+
let changed = false;
|
|
166
|
+
|
|
167
|
+
for (const action of report.workflow?.actions || []) {
|
|
168
|
+
if (action.status !== "suggested") continue;
|
|
169
|
+
if (action.type === "reopen_issue" && workflowConfig.github.reopenIssues !== "never" && action.issue) {
|
|
170
|
+
await transport.updateIssueState(action.issue.repo, action.issue.number, "open");
|
|
171
|
+
changed = true;
|
|
172
|
+
actions.push({ ...action, status: "applied" });
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
if (workflowConfig.github.closeIssues === "after_stability_window") {
|
|
178
|
+
const requiredCleanRuns = workflowConfig.stability?.cleanRunsBeforeClose || 3;
|
|
179
|
+
for (const regressionCase of cases) {
|
|
180
|
+
if (regressionCase.state !== "fixed_pending_verification" || !regressionCase.issue) continue;
|
|
181
|
+
if (Number(regressionCase.lifecycle?.cleanRunCount || 0) < requiredCleanRuns) continue;
|
|
182
|
+
await transport.updateIssueState(regressionCase.issue.repo, regressionCase.issue.number, "closed");
|
|
183
|
+
regressionCase.state = "resolved";
|
|
184
|
+
regressionCase.lifecycle = {
|
|
185
|
+
...(regressionCase.lifecycle || {}),
|
|
186
|
+
resolvedAt: new Date().toISOString(),
|
|
187
|
+
};
|
|
188
|
+
changed = true;
|
|
189
|
+
actions.push({
|
|
190
|
+
type: "close_issue",
|
|
191
|
+
caseId: regressionCase.id,
|
|
192
|
+
issue: regressionCase.issue,
|
|
193
|
+
reason: "stability_window_clean",
|
|
194
|
+
status: "applied",
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
for (const action of report.workflow?.actions || []) {
|
|
200
|
+
if (action.type !== "create_case" || action.status !== "applied") continue;
|
|
201
|
+
const regressionCase = caseById.get(action.caseId);
|
|
202
|
+
if (!regressionCase || regressionCase.issue || workflowConfig.github.createIssues === "never") continue;
|
|
203
|
+
if (regressionCase.classification === "infra" && workflowConfig.evidence?.requireForIssueCreation) continue;
|
|
204
|
+
const repo = caseStore.issueRepo;
|
|
205
|
+
if (!repo) continue;
|
|
206
|
+
const issue = await transport.createIssue(repo, {
|
|
207
|
+
title: regressionCase.summary,
|
|
208
|
+
body: buildIssueBody(regressionCase, action),
|
|
209
|
+
labels: workflowConfig.github.issueLabels || workflowConfig.github.bugLabels || ["bug"],
|
|
210
|
+
});
|
|
211
|
+
if (issue?.number) {
|
|
212
|
+
regressionCase.issue = { repo, number: issue.number };
|
|
213
|
+
regressionCase.state = "active";
|
|
214
|
+
changed = true;
|
|
215
|
+
actions.push({
|
|
216
|
+
type: "create_issue",
|
|
217
|
+
caseId: regressionCase.id,
|
|
218
|
+
issue: regressionCase.issue,
|
|
219
|
+
reason: "new_confirmed_regression",
|
|
220
|
+
status: "applied",
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return { changed };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async function maybeCreateGitHubTransport(workflowConfig, env) {
|
|
228
|
+
if (workflowConfig?.github?.read === false && !workflowConfig?.github?.write) return null;
|
|
229
|
+
return createDefaultGitHubIssueTransport(env, {
|
|
230
|
+
apiBaseUrl: workflowConfig?.github?.apiBaseUrl,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function buildIssueBody(regressionCase, action) {
|
|
235
|
+
return [
|
|
236
|
+
regressionCase.cause,
|
|
237
|
+
"",
|
|
238
|
+
"Testkit evidence:",
|
|
239
|
+
...(action.tests || []).map((test) => `- ${test.service} ${test.type} ${test.path}: ${test.status}`),
|
|
240
|
+
].join("\n");
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function collectRefs(refs, text, pattern, kind) {
|
|
244
|
+
for (const match of String(text || "").matchAll(pattern)) {
|
|
245
|
+
refs.push({ raw: match[1], kind });
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function normalizeIssueRef(ref, defaultRepo) {
|
|
250
|
+
const match = String(ref.raw || "").match(/^(?:(?<repo>[\w.-]+\/[\w.-]+))?#(?<number>\d+)$/u);
|
|
251
|
+
if (!match) return null;
|
|
252
|
+
const repo = match.groups.repo || defaultRepo;
|
|
253
|
+
const number = Number(match.groups.number);
|
|
254
|
+
if (!repo || !Number.isInteger(number)) return null;
|
|
255
|
+
return { kind: ref.kind, repo, number };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function summarizeWorkflowActions(actions) {
|
|
259
|
+
const summary = { applied: 0, suggested: 0, byType: {} };
|
|
260
|
+
for (const action of actions) {
|
|
261
|
+
if (action.status === "applied") summary.applied += 1;
|
|
262
|
+
if (action.status === "suggested") summary.suggested += 1;
|
|
263
|
+
summary.byType[action.type] = (summary.byType[action.type] || 0) + 1;
|
|
264
|
+
}
|
|
265
|
+
return summary;
|
|
266
|
+
}
|