@elench/testkit 0.1.149 → 0.1.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -9
- package/lib/cli/assistant/view-model.mjs +1 -1
- package/lib/cli/components/blocks/run-tree.mjs +1 -1
- package/lib/cli/renderers/run/events.mjs +4 -3
- package/lib/cli/renderers/run/failure.mjs +2 -2
- package/lib/cli/renderers/run/inline-detail.mjs +2 -2
- package/lib/cli/renderers/run/interactive.mjs +2 -2
- package/lib/cli/renderers/run/text-reporter.mjs +9 -9
- package/lib/cli/state/run/model.mjs +7 -7
- package/lib/cli/state/run/state.mjs +3 -3
- package/lib/cli/terminal/colors.mjs +1 -1
- package/lib/config/database-materialization.mjs +25 -0
- package/lib/config/database.mjs +30 -0
- package/lib/config/index.mjs +47 -1
- package/lib/config/runtime.mjs +130 -0
- package/lib/config-api/index.d.ts +28 -0
- package/lib/config-api/index.mjs +6 -0
- package/lib/database/cleanup.mjs +76 -1
- package/lib/database/constants.mjs +3 -0
- package/lib/database/index.mjs +6 -0
- package/lib/database/local-postgres.mjs +123 -4
- package/lib/database/naming.mjs +7 -0
- package/lib/database/resource-postgres.mjs +13 -0
- package/lib/database/state-files.mjs +17 -0
- package/lib/docker-compat/matrix.mjs +5 -3
- package/lib/kiln/client.mjs +8 -0
- package/lib/local/kiln-driver.mjs +96 -68
- package/lib/ownership/docker.mjs +67 -1
- package/lib/regressions/github-transport.mjs +178 -4
- package/lib/regressions/github.mjs +52 -16
- package/lib/regressions/index.d.ts +58 -29
- package/lib/regressions/index.mjs +171 -58
- package/lib/regressions/workflow.mjs +266 -0
- package/lib/results/artifacts.mjs +8 -7
- package/lib/runner/formatting.mjs +17 -16
- package/lib/runner/orchestrator.mjs +6 -5
- package/lib/runner/planning.mjs +40 -0
- package/lib/runner/regressions.mjs +183 -33
- package/lib/runner/reporting.mjs +1 -1
- package/lib/runner/run-finalization.mjs +34 -4
- package/lib/runner/runtime-manager.mjs +91 -10
- package/lib/runner/scheduler/index.mjs +30 -1
- package/lib/runtime/index.d.ts +5 -5
- package/lib/runtime-src/k6/http.js +11 -11
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/node_modules/es-toolkit/CHANGELOG.md +801 -0
- package/node_modules/es-toolkit/src/compat/_internal/Equals.d.ts +1 -0
- package/node_modules/es-toolkit/src/compat/_internal/IsWritable.d.ts +3 -0
- package/node_modules/es-toolkit/src/compat/_internal/MutableList.d.ts +4 -0
- package/node_modules/es-toolkit/src/compat/_internal/RejectReadonly.d.ts +4 -0
- package/node_modules/esprima/ChangeLog +235 -0
- package/package.json +6 -5
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts +0 -188
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js +0 -293
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/package.json +0 -25
|
@@ -295,23 +295,24 @@ function formatDiagnosis(diagnosis) {
|
|
|
295
295
|
if (diagnosis.classifications?.length) {
|
|
296
296
|
lines.push(`classification: ${diagnosis.classifications.join(", ")}`);
|
|
297
297
|
}
|
|
298
|
-
for (const entry of diagnosis.
|
|
299
|
-
lines.push(`
|
|
298
|
+
for (const entry of diagnosis.cases || []) {
|
|
299
|
+
lines.push(`case: ${entry.id}`);
|
|
300
|
+
if (entry.issue) lines.push(`issue: ${entry.issue.repo}#${entry.issue.number}`);
|
|
300
301
|
lines.push(`summary: ${entry.summary}`);
|
|
301
302
|
if (entry.github?.url) lines.push(`url: ${entry.github.url}`);
|
|
302
303
|
if (entry.github?.state) {
|
|
303
304
|
lines.push(`github state: ${entry.github.state}${entry.github.cached ? " (cache)" : ""}`);
|
|
304
305
|
}
|
|
305
|
-
if (entry.syncStatus) lines.push(`
|
|
306
|
-
if (entry.
|
|
307
|
-
for (const finding of entry.
|
|
308
|
-
lines.push(`
|
|
306
|
+
if (entry.syncStatus) lines.push(`case status: ${entry.syncStatus}`);
|
|
307
|
+
if (entry.syncFindings?.length) {
|
|
308
|
+
for (const finding of entry.syncFindings.slice(0, 3)) {
|
|
309
|
+
lines.push(`case finding: ${finding.message}`);
|
|
309
310
|
}
|
|
310
311
|
}
|
|
311
312
|
break;
|
|
312
313
|
}
|
|
313
314
|
if (diagnosis.status === "new_regression") {
|
|
314
|
-
lines.push("next: review .testkit/results/latest.json drafts.newRegressions for a prepared
|
|
315
|
+
lines.push("next: review .testkit/results/latest.json drafts.newRegressions for a prepared case");
|
|
315
316
|
}
|
|
316
317
|
return lines;
|
|
317
318
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { findMatchingRegressionCases } from "../regressions/index.mjs";
|
|
2
2
|
import { buildRegressionSyncSummaryLines } from "../regressions/github.mjs";
|
|
3
3
|
|
|
4
4
|
export function formatDuration(durationMs) {
|
|
@@ -70,8 +70,8 @@ export function buildRunSummaryData(results, durationMs, regressionReport = null
|
|
|
70
70
|
newRegressions: regressionSummary?.newRegressions || 0,
|
|
71
71
|
knownRegressions: regressionSummary?.knownRegressions || 0,
|
|
72
72
|
fixedKnownRegressions: regressionSummary?.fixedKnownRegressions || 0,
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
staleCases: regressionSummary?.staleCases || 0,
|
|
74
|
+
syncUnavailable: Boolean(regressionSummary?.syncUnavailable),
|
|
75
75
|
usedStaleCache: Boolean(regressionSummary?.usedStaleCache),
|
|
76
76
|
};
|
|
77
77
|
}
|
|
@@ -100,11 +100,11 @@ export function buildCompactRunSummaryLines(
|
|
|
100
100
|
if (summary.fixedKnownRegressions > 0) {
|
|
101
101
|
lines.push(`Fixed known regressions: ${summary.fixedKnownRegressions}`);
|
|
102
102
|
}
|
|
103
|
-
if (summary.
|
|
104
|
-
lines.push(`
|
|
103
|
+
if (summary.staleCases > 0) {
|
|
104
|
+
lines.push(`Case store stale: ${summary.staleCases}`);
|
|
105
105
|
}
|
|
106
|
-
if (summary.
|
|
107
|
-
lines.push("
|
|
106
|
+
if (summary.syncUnavailable) {
|
|
107
|
+
lines.push("Case store sync unavailable");
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
lines.push("");
|
|
@@ -186,11 +186,11 @@ export function buildDebugRunSummaryLines(results, durationMs, regressionReport
|
|
|
186
186
|
}
|
|
187
187
|
|
|
188
188
|
const regressionSyncLines = buildRegressionSyncSummaryLines(
|
|
189
|
-
regressionReport?.
|
|
189
|
+
regressionReport?.caseStore?.configured ? {
|
|
190
190
|
summary: {
|
|
191
191
|
byCode: {
|
|
192
|
-
closed_but_failing: regressionReport.summary.
|
|
193
|
-
validation_unavailable: regressionReport.summary.
|
|
192
|
+
closed_but_failing: regressionReport.summary.staleCases,
|
|
193
|
+
validation_unavailable: regressionReport.summary.syncUnavailable ? 1 : 0,
|
|
194
194
|
used_stale_cache: regressionReport.summary.usedStaleCache ? 1 : 0,
|
|
195
195
|
},
|
|
196
196
|
},
|
|
@@ -209,7 +209,7 @@ export function buildDebugRunSummaryLines(results, durationMs, regressionReport
|
|
|
209
209
|
return lines;
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
export function buildFailurePresentation(fileSummary,
|
|
212
|
+
export function buildFailurePresentation(fileSummary, regressionCaseStore = null) {
|
|
213
213
|
const rankedDetails = rankFailureDetails(fileSummary.failureDetails || []);
|
|
214
214
|
const primaryDetail = rankedDetails[0] || null;
|
|
215
215
|
const fallbackMessages = rankedDetails
|
|
@@ -221,7 +221,7 @@ export function buildFailurePresentation(fileSummary, regressionCatalog = null)
|
|
|
221
221
|
const responseLine = formatFailureResponsePreview(primaryDetail);
|
|
222
222
|
if (responseLine) details.push(responseLine);
|
|
223
223
|
|
|
224
|
-
const regressionLine = formatInlineRegressionLine(fileSummary,
|
|
224
|
+
const regressionLine = formatInlineRegressionLine(fileSummary, regressionCaseStore);
|
|
225
225
|
if (regressionLine) details.push(regressionLine);
|
|
226
226
|
|
|
227
227
|
const requestLine = formatFailureRequestHint(primaryDetail);
|
|
@@ -318,13 +318,14 @@ function formatFailureRequestHint(detail) {
|
|
|
318
318
|
return `request: ${method} ${path}`;
|
|
319
319
|
}
|
|
320
320
|
|
|
321
|
-
function formatInlineRegressionLine(fileSummary,
|
|
322
|
-
if (!
|
|
323
|
-
const matches =
|
|
321
|
+
function formatInlineRegressionLine(fileSummary, regressionCaseStore) {
|
|
322
|
+
if (!regressionCaseStore) return "regression: new";
|
|
323
|
+
const matches = findMatchingRegressionCases(regressionCaseStore, fileSummary);
|
|
324
324
|
if (matches.length === 0) return "regression: new";
|
|
325
325
|
|
|
326
326
|
const entry = matches[0];
|
|
327
|
-
|
|
327
|
+
const issue = entry.issue ? ` #${entry.issue.number}` : "";
|
|
328
|
+
return `regression: known${issue} ${entry.classification}`;
|
|
328
329
|
}
|
|
329
330
|
|
|
330
331
|
function isThresholdWrapperMessage(message) {
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
recordGraphError,
|
|
13
13
|
recordTaskOutcome,
|
|
14
14
|
} from "./results.mjs";
|
|
15
|
-
import {
|
|
15
|
+
import { loadRegressionCaseStoreConfig } from "./regressions.mjs";
|
|
16
16
|
import { formatError } from "./formatting.mjs";
|
|
17
17
|
import {
|
|
18
18
|
loadTimings,
|
|
@@ -58,12 +58,12 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
58
58
|
},
|
|
59
59
|
testkitVersion: readPackageMetadata().version,
|
|
60
60
|
};
|
|
61
|
-
const
|
|
61
|
+
const regressionCaseStore = loadRegressionCaseStoreConfig(
|
|
62
62
|
productDir,
|
|
63
63
|
configs[0]?.testkit?.regressions || null
|
|
64
64
|
);
|
|
65
65
|
const reporter = opts.reporter || null;
|
|
66
|
-
reporter?.
|
|
66
|
+
reporter?.setRegressionCaseStore?.(regressionCaseStore);
|
|
67
67
|
const logRegistry = createRunLogRegistry(productDir);
|
|
68
68
|
const workerState = {
|
|
69
69
|
workerCount: 0,
|
|
@@ -128,7 +128,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
128
128
|
const history = loadHistory(productDir);
|
|
129
129
|
const graphs = buildRuntimeGraphs(executedPlans);
|
|
130
130
|
const queue = buildScheduledQueue(executedPlans, graphs, { timings, history });
|
|
131
|
-
runSelection.planning = buildRunPlanningMetadata(queue);
|
|
131
|
+
runSelection.planning = buildRunPlanningMetadata(queue, graphs);
|
|
132
132
|
writeLiveSnapshot();
|
|
133
133
|
reporter?.setTotalFileCount?.(queue.length);
|
|
134
134
|
for (const task of queue) {
|
|
@@ -213,8 +213,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
213
213
|
metadata,
|
|
214
214
|
logRegistry,
|
|
215
215
|
setupRegistry,
|
|
216
|
-
|
|
216
|
+
regressionCaseStore,
|
|
217
217
|
regressionSyncConfig: configs[0]?.testkit?.regressions?.sync || null,
|
|
218
|
+
regressionWorkflowConfig: configs[0]?.testkit?.regressions?.workflow || null,
|
|
218
219
|
telemetry,
|
|
219
220
|
reporter,
|
|
220
221
|
writeStatus: opts.writeStatus,
|
package/lib/runner/planning.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
matchesSuiteSelectors,
|
|
4
4
|
suiteSelectionType,
|
|
5
5
|
} from "./suite-selection.mjs";
|
|
6
|
+
import { materializeDatabaseConfig } from "../config/database-materialization.mjs";
|
|
6
7
|
|
|
7
8
|
const TYPE_ORDER = ["ui", "e2e", "scenario", "integration", "dal", "load"];
|
|
8
9
|
|
|
@@ -122,6 +123,7 @@ export function buildRuntimeGraphs(servicePlans) {
|
|
|
122
123
|
dirName: buildGraphDirName(plan.runtimeNames),
|
|
123
124
|
instanceCount: plan.config.testkit.runtime.instances,
|
|
124
125
|
maxConcurrentTasks: resolveGraphMaxConcurrentTasks(plan.runtimeConfigs),
|
|
126
|
+
resourceReservations: buildGraphResourceReservations(plan.runtimeConfigs),
|
|
125
127
|
};
|
|
126
128
|
graphs.push(graph);
|
|
127
129
|
graphByRuntimeKey.set(plan.runtimeKey, graph);
|
|
@@ -137,6 +139,44 @@ export function buildRuntimeGraphs(servicePlans) {
|
|
|
137
139
|
}));
|
|
138
140
|
}
|
|
139
141
|
|
|
142
|
+
export function buildGraphResourceReservations(runtimeConfigs) {
|
|
143
|
+
const reservations = new Map();
|
|
144
|
+
|
|
145
|
+
for (const config of runtimeConfigs) {
|
|
146
|
+
const db = materializeDatabaseConfig(config.testkit.database, config.name);
|
|
147
|
+
if (!db) continue;
|
|
148
|
+
const key = postgresBudgetKey(db);
|
|
149
|
+
const existing = reservations.get(key) || {
|
|
150
|
+
key,
|
|
151
|
+
kind: "postgres",
|
|
152
|
+
scope: db.provider === "resource" ? "resource" : "local",
|
|
153
|
+
resourceName: db.provider === "resource" ? db.resource : "local",
|
|
154
|
+
capacity: db.maxConnections,
|
|
155
|
+
reserved: db.reservedConnections || 0,
|
|
156
|
+
perRuntime: 0,
|
|
157
|
+
services: [],
|
|
158
|
+
};
|
|
159
|
+
existing.capacity = Math.max(existing.capacity || 0, db.maxConnections || 0);
|
|
160
|
+
existing.reserved = Math.max(existing.reserved || 0, db.reservedConnections || 0);
|
|
161
|
+
existing.perRuntime += db.runtimeConnections || 1;
|
|
162
|
+
existing.services.push(config.name);
|
|
163
|
+
reservations.set(key, existing);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return [...reservations.values()]
|
|
167
|
+
.map((reservation) => ({
|
|
168
|
+
...reservation,
|
|
169
|
+
available: Math.max(0, (reservation.capacity || 0) - (reservation.reserved || 0)),
|
|
170
|
+
services: [...new Set(reservation.services)].sort(),
|
|
171
|
+
}))
|
|
172
|
+
.sort((left, right) => left.key.localeCompare(right.key));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function postgresBudgetKey(db) {
|
|
176
|
+
if (db.provider === "resource") return `postgres:resource:${db.resource}`;
|
|
177
|
+
return "postgres:local";
|
|
178
|
+
}
|
|
179
|
+
|
|
140
180
|
export function claimNextTask(queue, preferredGraphKey, isRunnable = () => true) {
|
|
141
181
|
if (queue.length === 0) return null;
|
|
142
182
|
|
|
@@ -1,24 +1,27 @@
|
|
|
1
1
|
import {
|
|
2
2
|
buildRegressionFileIdentity,
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
findMatchingRegressionCases,
|
|
4
|
+
loadRegressionCaseStoreConfig,
|
|
5
|
+
writeRegressionCaseStoreDocument,
|
|
5
6
|
} from "../regressions/index.mjs";
|
|
6
7
|
|
|
7
|
-
export {
|
|
8
|
+
export { loadRegressionCaseStoreConfig };
|
|
8
9
|
|
|
9
10
|
export function applyRegressionAnalysisToArtifacts(
|
|
10
11
|
runArtifact,
|
|
11
12
|
statusArtifact,
|
|
12
|
-
|
|
13
|
+
regressionCaseStore,
|
|
13
14
|
regressionSync
|
|
14
15
|
) {
|
|
15
16
|
const runEntries = extractRunFileEntries(runArtifact);
|
|
16
17
|
const statusEntries = extractStatusFileEntries(statusArtifact);
|
|
17
18
|
const fileSummaries = new Map();
|
|
18
|
-
const syncById = new Map((regressionSync?.
|
|
19
|
-
const
|
|
19
|
+
const syncById = new Map((regressionSync?.cases || []).map((entry) => [entry.id, entry]));
|
|
20
|
+
const staleCaseIds = new Set();
|
|
20
21
|
const newRegressionDrafts = [];
|
|
21
22
|
const fixedRegressionDrafts = [];
|
|
23
|
+
const workflowActions = [];
|
|
24
|
+
let caseStoreChanged = false;
|
|
22
25
|
|
|
23
26
|
for (const entry of [...runEntries, ...statusEntries]) {
|
|
24
27
|
const key = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
|
|
@@ -37,23 +40,64 @@ export function applyRegressionAnalysisToArtifacts(
|
|
|
37
40
|
const diagnosesByFileKey = new Map();
|
|
38
41
|
|
|
39
42
|
for (const fileSummary of fileSummaries.values()) {
|
|
40
|
-
const diagnosis = buildFileDiagnosis(fileSummary,
|
|
43
|
+
const diagnosis = buildFileDiagnosis(fileSummary, regressionCaseStore, syncById);
|
|
41
44
|
const fileKey = buildRegressionFileIdentity(
|
|
42
45
|
fileSummary.service,
|
|
43
46
|
fileSummary.type,
|
|
44
47
|
fileSummary.path
|
|
45
48
|
);
|
|
46
49
|
diagnosesByFileKey.set(fileKey, diagnosis);
|
|
47
|
-
for (const entry of diagnosis.
|
|
48
|
-
if (entry.
|
|
49
|
-
|
|
50
|
+
for (const entry of diagnosis.cases) {
|
|
51
|
+
if (entry.syncFindings.length > 0) {
|
|
52
|
+
staleCaseIds.add(entry.id);
|
|
50
53
|
}
|
|
51
54
|
}
|
|
52
55
|
if (diagnosis.status === "new_regression") {
|
|
53
|
-
|
|
56
|
+
const draft = buildNewRegressionDraft(fileSummary, regressionCaseStore);
|
|
57
|
+
newRegressionDrafts.push(draft);
|
|
58
|
+
if (regressionCaseStore) {
|
|
59
|
+
const createdCase = createCaseFromDraft(draft);
|
|
60
|
+
if (!caseIdSet(regressionCaseStore).has(createdCase.id)) {
|
|
61
|
+
regressionCaseStore.cases.push(createdCase);
|
|
62
|
+
caseStoreChanged = true;
|
|
63
|
+
workflowActions.push({
|
|
64
|
+
type: "create_case",
|
|
65
|
+
caseId: createdCase.id,
|
|
66
|
+
reason: "new_regression",
|
|
67
|
+
status: "applied",
|
|
68
|
+
tests: [testRef(fileSummary)],
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
54
72
|
}
|
|
55
73
|
if (diagnosis.status === "fixed_known_regression") {
|
|
56
|
-
for (const entry of diagnosis.
|
|
74
|
+
for (const entry of diagnosis.cases) {
|
|
75
|
+
const sourceCase = findCaseById(regressionCaseStore, entry.id);
|
|
76
|
+
if (sourceCase) {
|
|
77
|
+
const nextLifecycle = {
|
|
78
|
+
...(sourceCase.lifecycle || {}),
|
|
79
|
+
lastVerifiedAt: new Date().toISOString(),
|
|
80
|
+
cleanRunCount: Number(sourceCase.lifecycle?.cleanRunCount || 0) + 1,
|
|
81
|
+
};
|
|
82
|
+
const previousState = JSON.stringify({
|
|
83
|
+
state: sourceCase.state,
|
|
84
|
+
lifecycle: sourceCase.lifecycle || {},
|
|
85
|
+
});
|
|
86
|
+
sourceCase.state = "fixed_pending_verification";
|
|
87
|
+
sourceCase.lifecycle = nextLifecycle;
|
|
88
|
+
caseStoreChanged = caseStoreChanged || previousState !== JSON.stringify({
|
|
89
|
+
state: sourceCase.state,
|
|
90
|
+
lifecycle: sourceCase.lifecycle || {},
|
|
91
|
+
});
|
|
92
|
+
workflowActions.push({
|
|
93
|
+
type: "mark_fixed_pending_verification",
|
|
94
|
+
caseId: entry.id,
|
|
95
|
+
issue: entry.issue,
|
|
96
|
+
reason: "matched_case_passed",
|
|
97
|
+
status: "applied",
|
|
98
|
+
tests: [testRef(fileSummary)],
|
|
99
|
+
});
|
|
100
|
+
}
|
|
57
101
|
fixedRegressionDrafts.push({
|
|
58
102
|
id: entry.id,
|
|
59
103
|
issue: entry.issue,
|
|
@@ -62,6 +106,31 @@ export function applyRegressionAnalysisToArtifacts(
|
|
|
62
106
|
});
|
|
63
107
|
}
|
|
64
108
|
}
|
|
109
|
+
if (diagnosis.status === "known_regression") {
|
|
110
|
+
for (const entry of diagnosis.cases) {
|
|
111
|
+
const sourceCase = findCaseById(regressionCaseStore, entry.id);
|
|
112
|
+
if (!sourceCase) continue;
|
|
113
|
+
const nextLifecycle = {
|
|
114
|
+
...(sourceCase.lifecycle || {}),
|
|
115
|
+
cleanRunCount: 0,
|
|
116
|
+
};
|
|
117
|
+
if (sourceCase.state !== "active" || JSON.stringify(sourceCase.lifecycle || {}) !== JSON.stringify(nextLifecycle)) {
|
|
118
|
+
sourceCase.state = "active";
|
|
119
|
+
sourceCase.lifecycle = nextLifecycle;
|
|
120
|
+
caseStoreChanged = true;
|
|
121
|
+
}
|
|
122
|
+
if (entry.syncStatus === "closed_but_failing") {
|
|
123
|
+
workflowActions.push({
|
|
124
|
+
type: "reopen_issue",
|
|
125
|
+
caseId: entry.id,
|
|
126
|
+
issue: entry.issue,
|
|
127
|
+
reason: "closed_issue_reproduced",
|
|
128
|
+
status: "suggested",
|
|
129
|
+
tests: [testRef(fileSummary)],
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
65
134
|
}
|
|
66
135
|
|
|
67
136
|
for (const entry of [...runEntries, ...statusEntries]) {
|
|
@@ -74,32 +143,47 @@ export function applyRegressionAnalysisToArtifacts(
|
|
|
74
143
|
const summaryTests = statusArtifact?.tests || runEntries;
|
|
75
144
|
const report = buildRegressionReport(
|
|
76
145
|
summaryTests,
|
|
77
|
-
|
|
146
|
+
regressionCaseStore,
|
|
78
147
|
regressionSync,
|
|
79
148
|
diagnosesByFileKey,
|
|
80
|
-
|
|
149
|
+
staleCaseIds,
|
|
81
150
|
newRegressionDrafts,
|
|
82
|
-
fixedRegressionDrafts
|
|
151
|
+
fixedRegressionDrafts,
|
|
152
|
+
workflowActions
|
|
83
153
|
);
|
|
84
154
|
|
|
85
155
|
runArtifact.regressions = report;
|
|
156
|
+
runArtifact.summary = {
|
|
157
|
+
...(runArtifact.summary || {}),
|
|
158
|
+
regressions: report.summary,
|
|
159
|
+
};
|
|
86
160
|
if (statusArtifact) {
|
|
87
161
|
statusArtifact.regressions = report;
|
|
162
|
+
statusArtifact.summary = {
|
|
163
|
+
...(statusArtifact.summary || {}),
|
|
164
|
+
regressions: report.summary,
|
|
165
|
+
};
|
|
88
166
|
}
|
|
89
167
|
|
|
90
|
-
return {
|
|
168
|
+
return {
|
|
169
|
+
runArtifact,
|
|
170
|
+
statusArtifact,
|
|
171
|
+
regressionReport: report,
|
|
172
|
+
regressionCaseStore: regressionCaseStore,
|
|
173
|
+
regressionCaseStoreChanged: caseStoreChanged,
|
|
174
|
+
};
|
|
91
175
|
}
|
|
92
176
|
|
|
93
|
-
function buildFileDiagnosis(fileSummary,
|
|
94
|
-
const matchedEntries =
|
|
95
|
-
?
|
|
177
|
+
function buildFileDiagnosis(fileSummary, regressionCaseStore, syncById) {
|
|
178
|
+
const matchedEntries = regressionCaseStore
|
|
179
|
+
? findMatchingRegressionCases(regressionCaseStore, fileSummary)
|
|
96
180
|
: [];
|
|
97
181
|
const status = resolveDiagnosisStatus(fileSummary.status, matchedEntries.length);
|
|
98
182
|
|
|
99
183
|
return {
|
|
100
184
|
status,
|
|
101
185
|
classifications: [...new Set(matchedEntries.map((entry) => entry.classification))].sort(),
|
|
102
|
-
|
|
186
|
+
cases: matchedEntries.map((entry) => toDiagnosisEntry(entry, syncById.get(entry.id) || null)),
|
|
103
187
|
};
|
|
104
188
|
}
|
|
105
189
|
|
|
@@ -119,6 +203,7 @@ function resolveDiagnosisStatus(fileStatus, matchCount) {
|
|
|
119
203
|
function toDiagnosisEntry(entry, syncEntry) {
|
|
120
204
|
return {
|
|
121
205
|
id: entry.id,
|
|
206
|
+
state: entry.state,
|
|
122
207
|
classification: entry.classification,
|
|
123
208
|
issue: entry.issue,
|
|
124
209
|
summary: entry.summary,
|
|
@@ -126,25 +211,26 @@ function toDiagnosisEntry(entry, syncEntry) {
|
|
|
126
211
|
lastReviewedAt: entry.lastReviewedAt,
|
|
127
212
|
github: syncEntry?.github || null,
|
|
128
213
|
syncStatus: syncEntry?.status || null,
|
|
129
|
-
|
|
214
|
+
syncFindings: Array.isArray(syncEntry?.findings) ? syncEntry.findings : [],
|
|
130
215
|
};
|
|
131
216
|
}
|
|
132
217
|
|
|
133
218
|
function buildRegressionReport(
|
|
134
219
|
tests,
|
|
135
|
-
|
|
220
|
+
regressionCaseStore,
|
|
136
221
|
regressionSync,
|
|
137
222
|
diagnosesByFileKey,
|
|
138
|
-
|
|
223
|
+
staleCaseIds,
|
|
139
224
|
newRegressionDrafts,
|
|
140
|
-
fixedRegressionDrafts
|
|
225
|
+
fixedRegressionDrafts,
|
|
226
|
+
workflowActions = []
|
|
141
227
|
) {
|
|
142
228
|
const summary = {
|
|
143
229
|
newRegressions: 0,
|
|
144
230
|
knownRegressions: 0,
|
|
145
231
|
fixedKnownRegressions: 0,
|
|
146
|
-
|
|
147
|
-
|
|
232
|
+
staleCases: staleCaseIds.size,
|
|
233
|
+
syncUnavailable: (regressionSync?.summary?.byCode?.validation_unavailable || 0) > 0,
|
|
148
234
|
usedStaleCache: (regressionSync?.summary?.byCode?.used_stale_cache || 0) > 0,
|
|
149
235
|
};
|
|
150
236
|
|
|
@@ -169,10 +255,14 @@ function buildRegressionReport(
|
|
|
169
255
|
|
|
170
256
|
return {
|
|
171
257
|
summary,
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
258
|
+
workflow: {
|
|
259
|
+
actions: workflowActions,
|
|
260
|
+
summary: summarizeWorkflowActions(workflowActions),
|
|
261
|
+
},
|
|
262
|
+
caseStore: {
|
|
263
|
+
configured: Boolean(regressionCaseStore),
|
|
264
|
+
caseCount: regressionCaseStore?.cases?.length || 0,
|
|
265
|
+
staleCases: (regressionSync?.cases || []).filter((entry) =>
|
|
176
266
|
Array.isArray(entry.findings) && entry.findings.length > 0
|
|
177
267
|
),
|
|
178
268
|
findings: regressionSync?.findings || [],
|
|
@@ -180,7 +270,7 @@ function buildRegressionReport(
|
|
|
180
270
|
mode: regressionSync?.mode || null,
|
|
181
271
|
checkedAt: regressionSync?.checkedAt || null,
|
|
182
272
|
usedStaleCache: summary.usedStaleCache,
|
|
183
|
-
unavailable: summary.
|
|
273
|
+
unavailable: summary.syncUnavailable,
|
|
184
274
|
},
|
|
185
275
|
},
|
|
186
276
|
drafts: {
|
|
@@ -190,12 +280,12 @@ function buildRegressionReport(
|
|
|
190
280
|
};
|
|
191
281
|
}
|
|
192
282
|
|
|
193
|
-
function buildNewRegressionDraft(fileSummary,
|
|
283
|
+
function buildNewRegressionDraft(fileSummary, regressionCaseStore) {
|
|
194
284
|
return {
|
|
195
285
|
id: suggestRegressionId(fileSummary),
|
|
196
286
|
classification: suggestRegressionClassification(fileSummary),
|
|
197
287
|
issue: {
|
|
198
|
-
repo:
|
|
288
|
+
repo: regressionCaseStore?.issueRepo || null,
|
|
199
289
|
number: null,
|
|
200
290
|
},
|
|
201
291
|
summary: suggestRegressionSummary(fileSummary),
|
|
@@ -211,6 +301,29 @@ function buildNewRegressionDraft(fileSummary, regressionCatalog) {
|
|
|
211
301
|
};
|
|
212
302
|
}
|
|
213
303
|
|
|
304
|
+
function createCaseFromDraft(draft) {
|
|
305
|
+
const now = new Date().toISOString();
|
|
306
|
+
return {
|
|
307
|
+
id: draft.id,
|
|
308
|
+
state: "untriaged",
|
|
309
|
+
classification: draft.classification,
|
|
310
|
+
issue: null,
|
|
311
|
+
summary: draft.summary,
|
|
312
|
+
cause: draft.cause,
|
|
313
|
+
lastReviewedAt: now.slice(0, 10),
|
|
314
|
+
fingerprints: draft.fingerprints,
|
|
315
|
+
lifecycle: {
|
|
316
|
+
firstSeenAt: now,
|
|
317
|
+
lastSeenAt: now,
|
|
318
|
+
cleanRunCount: 0,
|
|
319
|
+
},
|
|
320
|
+
coverage: {
|
|
321
|
+
required: true,
|
|
322
|
+
status: "covered_reproducing",
|
|
323
|
+
},
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
214
327
|
function suggestRegressionId(fileSummary) {
|
|
215
328
|
const base = `${fileSummary.service}-${fileSummary.type}-${fileSummary.path}`
|
|
216
329
|
.toLowerCase()
|
|
@@ -269,6 +382,43 @@ function dedupeDraftsById(entries) {
|
|
|
269
382
|
return [...map.values()];
|
|
270
383
|
}
|
|
271
384
|
|
|
385
|
+
function caseIdSet(regressionCaseStore) {
|
|
386
|
+
return new Set((regressionCaseStore?.cases || []).map((entry) => entry.id));
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function findCaseById(regressionCaseStore, id) {
|
|
390
|
+
return (regressionCaseStore?.cases || []).find((entry) => entry.id === id) || null;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function testRef(fileSummary) {
|
|
394
|
+
return {
|
|
395
|
+
service: fileSummary.service,
|
|
396
|
+
type: fileSummary.type,
|
|
397
|
+
path: fileSummary.path,
|
|
398
|
+
status: fileSummary.status,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function summarizeWorkflowActions(actions) {
|
|
403
|
+
const summary = {
|
|
404
|
+
applied: 0,
|
|
405
|
+
suggested: 0,
|
|
406
|
+
byType: {},
|
|
407
|
+
};
|
|
408
|
+
for (const action of actions) {
|
|
409
|
+
if (action.status === "applied") summary.applied += 1;
|
|
410
|
+
if (action.status === "suggested") summary.suggested += 1;
|
|
411
|
+
summary.byType[action.type] = (summary.byType[action.type] || 0) + 1;
|
|
412
|
+
}
|
|
413
|
+
return summary;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
export function persistRegressionCaseStoreIfChanged(productDir, regressionCaseStore, changed) {
|
|
417
|
+
if (!changed || !regressionCaseStore?.__filePath) return false;
|
|
418
|
+
writeRegressionCaseStoreDocument(regressionCaseStore.__filePath, regressionCaseStore);
|
|
419
|
+
return true;
|
|
420
|
+
}
|
|
421
|
+
|
|
272
422
|
function extractRunFileEntries(runArtifact) {
|
|
273
423
|
const entries = [];
|
|
274
424
|
|
package/lib/runner/reporting.mjs
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
applyRegressionAnalysisToArtifacts,
|
|
4
|
+
persistRegressionCaseStoreIfChanged,
|
|
5
|
+
} from "./regressions.mjs";
|
|
3
6
|
import { writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
|
|
4
7
|
import { summarizeDbBackend } from "./results.mjs";
|
|
5
8
|
import { formatError } from "./formatting.mjs";
|
|
6
9
|
import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
|
|
7
10
|
import { loadHistory, saveHistory, updateHistoryFromRunArtifact } from "../history/index.mjs";
|
|
8
11
|
import { shouldFailRegressionSync, validateRegressionIssues } from "../regressions/github.mjs";
|
|
12
|
+
import { executeRegressionWorkflowActions } from "../regressions/workflow.mjs";
|
|
9
13
|
|
|
10
14
|
export async function finalizeRunArtifacts({
|
|
11
15
|
productDir,
|
|
@@ -22,8 +26,9 @@ export async function finalizeRunArtifacts({
|
|
|
22
26
|
metadata,
|
|
23
27
|
logRegistry,
|
|
24
28
|
setupRegistry,
|
|
25
|
-
|
|
29
|
+
regressionCaseStore,
|
|
26
30
|
regressionSyncConfig,
|
|
31
|
+
regressionWorkflowConfig,
|
|
27
32
|
telemetry,
|
|
28
33
|
reporter,
|
|
29
34
|
writeStatus,
|
|
@@ -65,7 +70,7 @@ export async function finalizeRunArtifacts({
|
|
|
65
70
|
: null;
|
|
66
71
|
const regressionSync = await validateRegressionIssues({
|
|
67
72
|
productDir,
|
|
68
|
-
document:
|
|
73
|
+
document: regressionCaseStore,
|
|
69
74
|
runArtifact,
|
|
70
75
|
statusArtifact,
|
|
71
76
|
config: regressionSyncConfig,
|
|
@@ -74,14 +79,39 @@ export async function finalizeRunArtifacts({
|
|
|
74
79
|
const enrichedArtifacts = applyRegressionAnalysisToArtifacts(
|
|
75
80
|
runArtifact,
|
|
76
81
|
statusArtifact,
|
|
77
|
-
|
|
82
|
+
regressionCaseStore,
|
|
78
83
|
regressionSync
|
|
79
84
|
);
|
|
85
|
+
const workflowResult = await executeRegressionWorkflowActions({
|
|
86
|
+
productDir,
|
|
87
|
+
caseStore: enrichedArtifacts.regressionCaseStore,
|
|
88
|
+
report: enrichedArtifacts.regressionReport,
|
|
89
|
+
workflowConfig: regressionWorkflowConfig,
|
|
90
|
+
gitMetadata: metadata.git,
|
|
91
|
+
});
|
|
92
|
+
const caseStoreChanged = enrichedArtifacts.regressionCaseStoreChanged || workflowResult.changed;
|
|
93
|
+
enrichedArtifacts.runArtifact.regressions = enrichedArtifacts.regressionReport;
|
|
94
|
+
enrichedArtifacts.runArtifact.summary = {
|
|
95
|
+
...(enrichedArtifacts.runArtifact.summary || {}),
|
|
96
|
+
regressions: enrichedArtifacts.regressionReport.summary,
|
|
97
|
+
};
|
|
98
|
+
if (enrichedArtifacts.statusArtifact) {
|
|
99
|
+
enrichedArtifacts.statusArtifact.regressions = enrichedArtifacts.regressionReport;
|
|
100
|
+
enrichedArtifacts.statusArtifact.summary = {
|
|
101
|
+
...(enrichedArtifacts.statusArtifact.summary || {}),
|
|
102
|
+
regressions: enrichedArtifacts.regressionReport.summary,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
80
105
|
|
|
81
106
|
writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
|
|
82
107
|
if (writeStatus) {
|
|
83
108
|
writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
|
|
84
109
|
}
|
|
110
|
+
persistRegressionCaseStoreIfChanged(
|
|
111
|
+
productDir,
|
|
112
|
+
enrichedArtifacts.regressionCaseStore,
|
|
113
|
+
caseStoreChanged
|
|
114
|
+
);
|
|
85
115
|
const nextHistory = updateHistoryFromRunArtifact(
|
|
86
116
|
loadHistory(productDir),
|
|
87
117
|
enrichedArtifacts.runArtifact,
|