@sentinelqa/playwright-reporter 0.1.51 → 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -107
- package/dist/localReport.js +360 -32
- package/dist/quickDiagnosis.d.ts +42 -2
- package/dist/quickDiagnosis.js +1587 -89
- package/dist/reporter.js +160 -31
- package/dist/runHistory.d.ts +3 -0
- package/dist/runHistory.js +11 -1
- package/dist/terminalSummary.js +115 -27
- package/package.json +2 -2
package/dist/reporter.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
const node_1 = require("@sentinelqa/uploader/node");
|
|
6
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
7
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
3
8
|
const env_1 = require("./env");
|
|
4
9
|
const quickDiagnosis_1 = require("./quickDiagnosis");
|
|
5
10
|
const terminalSummary_1 = require("./terminalSummary");
|
|
@@ -11,10 +16,107 @@ const colorize = (value, code) => {
|
|
|
11
16
|
return value;
|
|
12
17
|
return `\u001b[${code}m${value}\u001b[0m`;
|
|
13
18
|
};
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
19
|
+
const styleCritical = (value) => colorize(value, "1;31");
|
|
20
|
+
const styleWarning = (value) => colorize(value, "1;33");
|
|
21
|
+
const styleAction = (value) => colorize(value, "1;36");
|
|
22
|
+
const stylePrimary = (value) => colorize(value, "1;97");
|
|
23
|
+
const styleImportant = (value) => colorize(value, "97");
|
|
24
|
+
const styleSecondary = (value) => colorize(value, "2");
|
|
17
25
|
const divider = () => "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━";
|
|
26
|
+
const LABEL_WIDTH = 19;
|
|
27
|
+
const renderRow = (label, value, valueStyle, indent = "") => {
|
|
28
|
+
const paddedLabel = `${label}:`.padEnd(LABEL_WIDTH);
|
|
29
|
+
const styledValue = valueStyle ? valueStyle(value) : value;
|
|
30
|
+
return `${indent}${stylePrimary(paddedLabel)} ${styledValue}`;
|
|
31
|
+
};
|
|
32
|
+
const highlightCounts = (value) => value
|
|
33
|
+
.replace(/\b(\d+\s+tests?\s+failed)\b/gi, (_, match) => styleCritical(match))
|
|
34
|
+
.replace(/\b(\d+\s+real\s+issues?)\b/gi, (_, match) => styleWarning(match))
|
|
35
|
+
.replace(/\b(\d+\s+tests?)\b/gi, (_, match) => styleWarning(match))
|
|
36
|
+
.replace(/\b(\d+\s+previous\s+runs?)\b/gi, (_, match) => styleSecondary(match))
|
|
37
|
+
.replace(/\b(\d+\s+passing\s+runs?)\b/gi, (_, match) => styleSecondary(match))
|
|
38
|
+
.replace(/\b(\d+\s+newly\s+failing)\b/gi, (_, match) => styleWarning(match))
|
|
39
|
+
.replace(/\b(\d+\s+tests?\s+still\s+failing)\b/gi, (_, match) => styleWarning(match));
|
|
40
|
+
const styleIssueTitle = (value) => stylePrimary(value.replace(/\((\d+\s+tests?)\)/i, (_, match) => `(${styleWarning(match)})`));
|
|
41
|
+
const styleWhereValue = (value) => value.replace(/\b([A-Za-z0-9_.-]+\.[cm]?[jt]sx?:\d+(?::\d+)?)\b/g, (_, match) => styleImportant(match));
|
|
42
|
+
const formatCliLine = (line) => {
|
|
43
|
+
if (!line.trim())
|
|
44
|
+
return line;
|
|
45
|
+
if (line === divider())
|
|
46
|
+
return styleSecondary(line);
|
|
47
|
+
if (/^NEW FAILURE/.test(line))
|
|
48
|
+
return styleCritical(line);
|
|
49
|
+
if (/^RECURRING FAILURE/.test(line)) {
|
|
50
|
+
return line.replace(/RECURRING FAILURE/, styleWarning("RECURRING FAILURE")).replace(/\(([^)]+)\)/, (_, inner) => `(${styleSecondary(inner)})`);
|
|
51
|
+
}
|
|
52
|
+
if (/^All tests passed$/.test(line))
|
|
53
|
+
return stylePrimary(line);
|
|
54
|
+
if (/^❌\s+\d+\s+failures?/.test(line))
|
|
55
|
+
return styleCritical(line);
|
|
56
|
+
if (/^\d+\s+tests?\s+failed$/.test(line))
|
|
57
|
+
return styleCritical(line);
|
|
58
|
+
if (/^Collapsed into \d+\s+real issue/.test(line))
|
|
59
|
+
return highlightCounts(line);
|
|
60
|
+
if (/^\d+\s+tests?\s+in\s+/.test(line))
|
|
61
|
+
return line;
|
|
62
|
+
if (/^Issue(?:(?: \d+)?):/.test(line)) {
|
|
63
|
+
const [head, rest] = line.split(": ", 2);
|
|
64
|
+
return `${stylePrimary(head)}: ${styleIssueTitle(rest || "")}`;
|
|
65
|
+
}
|
|
66
|
+
const rowMatch = line.match(/^(\s*)(What broke|Why|Cause|Where|What changed|Changes since last run|Next|Likely fix|Expected|Received|Confidence|Impact|At risk|Why this matters|Recommendation|Last failure|Status|Artifacts ready|Failing step|Selector|Target state|Report|Blocker):\s*(.*)$/);
|
|
67
|
+
if (!rowMatch)
|
|
68
|
+
return line;
|
|
69
|
+
const [, indent, label, rawValue] = rowMatch;
|
|
70
|
+
const value = highlightCounts(rawValue);
|
|
71
|
+
switch (label) {
|
|
72
|
+
case "What broke":
|
|
73
|
+
return renderRow(label, value, undefined, indent);
|
|
74
|
+
case "Why":
|
|
75
|
+
return renderRow(label, value, undefined, indent);
|
|
76
|
+
case "Cause":
|
|
77
|
+
return renderRow(label, rawValue, undefined, indent);
|
|
78
|
+
case "Where":
|
|
79
|
+
return renderRow(label, rawValue, styleWhereValue, indent);
|
|
80
|
+
case "What changed":
|
|
81
|
+
case "Changes since last run":
|
|
82
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
83
|
+
case "Next":
|
|
84
|
+
case "Likely fix":
|
|
85
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
86
|
+
case "Report":
|
|
87
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
88
|
+
case "Expected":
|
|
89
|
+
return renderRow(label, rawValue, styleImportant, indent);
|
|
90
|
+
case "Received":
|
|
91
|
+
return renderRow(label, rawValue, styleCritical, indent);
|
|
92
|
+
case "Confidence":
|
|
93
|
+
return renderRow(label, rawValue, /high/i.test(rawValue) ? styleAction : /medium/i.test(rawValue) ? styleWarning : styleSecondary, indent);
|
|
94
|
+
case "Impact":
|
|
95
|
+
return renderRow(label, highlightCounts(rawValue), undefined, indent);
|
|
96
|
+
case "At risk":
|
|
97
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
98
|
+
case "Why this matters":
|
|
99
|
+
return renderRow(label, rawValue, undefined, indent);
|
|
100
|
+
case "Recommendation":
|
|
101
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
102
|
+
case "Last failure":
|
|
103
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
104
|
+
case "Status":
|
|
105
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
106
|
+
case "Artifacts ready":
|
|
107
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
108
|
+
case "Failing step":
|
|
109
|
+
return renderRow(label, rawValue, styleImportant, indent);
|
|
110
|
+
case "Selector":
|
|
111
|
+
return renderRow(label, rawValue, styleImportant, indent);
|
|
112
|
+
case "Blocker":
|
|
113
|
+
return renderRow(label, rawValue, styleCritical, indent);
|
|
114
|
+
case "Target state":
|
|
115
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
116
|
+
default:
|
|
117
|
+
return line;
|
|
118
|
+
}
|
|
119
|
+
};
|
|
18
120
|
const readFinalFailedCount = (playwrightJsonPath) => {
|
|
19
121
|
try {
|
|
20
122
|
const parsed = JSON.parse(require("node:fs").readFileSync(playwrightJsonPath, "utf8"));
|
|
@@ -83,22 +185,26 @@ class SentinelReporter {
|
|
|
83
185
|
!hasCiEnv &&
|
|
84
186
|
!localUploadEnabled;
|
|
85
187
|
const quickDiagnosis = (0, quickDiagnosis_1.buildQuickDiagnosis)(this.options.playwrightJsonPath);
|
|
188
|
+
const quickDiagnosisStructured = (0, quickDiagnosis_1.buildQuickDiagnosisStructured)(this.options.playwrightJsonPath);
|
|
189
|
+
const isSetupFailureDiagnosis = Boolean(quickDiagnosis?.lines.some((line) => /^Issue(?:(?: \d+)?): Playwright setup error/.test(line)));
|
|
86
190
|
const finalFailedCount = readFinalFailedCount(this.options.playwrightJsonPath);
|
|
87
191
|
const effectiveFailedCount = typeof finalFailedCount === "number" ? finalFailedCount : this.failedCount;
|
|
88
192
|
const passingSummary = effectiveFailedCount === 0
|
|
89
193
|
? (0, terminalSummary_1.buildPassingRunSummary)(this.options.playwrightJsonPath, { observedRunDurationMs: Date.now() - this.startedAt })
|
|
90
194
|
: null;
|
|
91
|
-
const failedRunHistory = effectiveFailedCount > 0
|
|
195
|
+
const failedRunHistory = effectiveFailedCount > 0 && !isSetupFailureDiagnosis
|
|
196
|
+
? (0, runHistory_1.buildFailedRunHistorySummary)(this.options.playwrightJsonPath)
|
|
197
|
+
: null;
|
|
92
198
|
if (effectiveFailedCount > 0) {
|
|
93
199
|
(0, terminalSummary_1.recordReporterHistorySnapshot)(this.options.playwrightJsonPath);
|
|
94
200
|
}
|
|
95
201
|
console.log("");
|
|
96
202
|
if (passingSummary) {
|
|
97
|
-
console.log(
|
|
98
|
-
console.log(divider());
|
|
203
|
+
console.log(stylePrimary("Sentinel run summary"));
|
|
204
|
+
console.log(styleSecondary(divider()));
|
|
99
205
|
console.log("");
|
|
100
206
|
for (const line of passingSummary.lines) {
|
|
101
|
-
console.log(line);
|
|
207
|
+
console.log(formatCliLine(line));
|
|
102
208
|
}
|
|
103
209
|
console.log("");
|
|
104
210
|
console.log(divider());
|
|
@@ -117,34 +223,56 @@ class SentinelReporter {
|
|
|
117
223
|
return;
|
|
118
224
|
}
|
|
119
225
|
if (quickDiagnosis?.lines.length) {
|
|
120
|
-
console.log(
|
|
121
|
-
console.log(divider());
|
|
226
|
+
console.log(stylePrimary("Sentinel diagnosis"));
|
|
227
|
+
console.log(styleSecondary(divider()));
|
|
122
228
|
console.log("");
|
|
123
|
-
if (failedRunHistory?.passStreakBeforeFailure && failedRunHistory.passStreakBeforeFailure > 0) {
|
|
124
|
-
console.log(`NEW FAILURE after ${failedRunHistory.passStreakBeforeFailure} passing runs`);
|
|
229
|
+
if (!isSetupFailureDiagnosis && failedRunHistory?.passStreakBeforeFailure && failedRunHistory.passStreakBeforeFailure > 0) {
|
|
230
|
+
console.log(formatCliLine(`NEW FAILURE after ${failedRunHistory.passStreakBeforeFailure} passing runs`));
|
|
125
231
|
console.log("");
|
|
126
232
|
}
|
|
127
|
-
else if ((failedRunHistory?.recurringCount || 0) > 0) {
|
|
128
|
-
console.log(`RECURRING FAILURE (${failedRunHistory?.recurringCount} previous runs)`);
|
|
233
|
+
else if (!isSetupFailureDiagnosis && (failedRunHistory?.recurringCount || 0) > 0) {
|
|
234
|
+
console.log(formatCliLine(`RECURRING FAILURE (${failedRunHistory?.recurringCount} previous runs)`));
|
|
235
|
+
if (failedRunHistory?.isDominantRecurringIssue && failedRunHistory?.dominantRecurringIssueTitle) {
|
|
236
|
+
console.log(formatCliLine(`Impact: ${failedRunHistory.dominantRecurringIssueTitle} is the most common recent failure in local history`));
|
|
237
|
+
}
|
|
129
238
|
console.log("");
|
|
130
239
|
}
|
|
131
240
|
for (const line of quickDiagnosis.lines) {
|
|
132
|
-
console.log(line);
|
|
241
|
+
console.log(formatCliLine(line));
|
|
133
242
|
}
|
|
134
243
|
console.log("");
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (failedRunHistory?.newFailures && failedRunHistory.newFailures > 0) {
|
|
138
|
-
console.log(`Impact: ${failedRunHistory.newFailures} newly failing in this run`);
|
|
139
|
-
console.log("");
|
|
140
|
-
}
|
|
141
|
-
else if (failedRunHistory?.stillFailing && failedRunHistory.stillFailing > 0) {
|
|
142
|
-
console.log(`Impact: ${failedRunHistory.stillFailing} tests still failing`);
|
|
244
|
+
if (!isSetupFailureDiagnosis) {
|
|
245
|
+
console.log(styleSecondary(divider()));
|
|
143
246
|
console.log("");
|
|
247
|
+
if (failedRunHistory?.newFailures && failedRunHistory.newFailures > 0) {
|
|
248
|
+
console.log(formatCliLine(`Impact: ${failedRunHistory.newFailures} newly failing in this run`));
|
|
249
|
+
console.log("");
|
|
250
|
+
}
|
|
251
|
+
else if (failedRunHistory?.stillFailing && failedRunHistory.stillFailing > 0) {
|
|
252
|
+
console.log(formatCliLine(`Impact: ${failedRunHistory.stillFailing} tests still failing`));
|
|
253
|
+
if (failedRunHistory?.isDominantRecurringIssue && failedRunHistory?.dominantRecurringIssueCount > 0) {
|
|
254
|
+
console.log(formatCliLine(`Impact: Seen in ${failedRunHistory.dominantRecurringIssueCount} recorded failed runs`));
|
|
255
|
+
}
|
|
256
|
+
console.log("");
|
|
257
|
+
}
|
|
144
258
|
}
|
|
145
259
|
}
|
|
146
|
-
|
|
260
|
+
if (isSetupFailureDiagnosis) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
console.log(styleSecondary("Uploading debug report..."));
|
|
147
264
|
console.log("");
|
|
265
|
+
let structuredDiagnosisPath = null;
|
|
266
|
+
if (quickDiagnosisStructured) {
|
|
267
|
+
try {
|
|
268
|
+
structuredDiagnosisPath = node_path_1.default.join(this.options.testResultsDir, "sentinel-structured-diagnosis.json");
|
|
269
|
+
node_fs_1.default.mkdirSync(this.options.testResultsDir, { recursive: true });
|
|
270
|
+
node_fs_1.default.writeFileSync(structuredDiagnosisPath, JSON.stringify(quickDiagnosisStructured, null, 2));
|
|
271
|
+
}
|
|
272
|
+
catch {
|
|
273
|
+
structuredDiagnosisPath = null;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
148
276
|
const upload = (await (0, node_1.runSentinelUpload)({
|
|
149
277
|
playwrightJsonPath: this.options.playwrightJsonPath,
|
|
150
278
|
playwrightReportDir: this.options.playwrightReportDir,
|
|
@@ -154,27 +282,28 @@ class SentinelReporter {
|
|
|
154
282
|
env: {
|
|
155
283
|
SENTINEL_REPORTER_PROJECT: this.options.project || undefined,
|
|
156
284
|
SENTINEL_REPORTER_SILENT: "1",
|
|
285
|
+
SENTINEL_STRUCTURED_DIAGNOSIS_PATH: structuredDiagnosisPath || undefined,
|
|
157
286
|
SENTINEL_UPLOAD_LOCAL: usingImplicitLocalPublicMode ? "1" : process.env.SENTINEL_UPLOAD_LOCAL
|
|
158
287
|
}
|
|
159
288
|
}));
|
|
160
289
|
if (upload.exitCode !== 0) {
|
|
161
|
-
|
|
290
|
+
return;
|
|
162
291
|
}
|
|
163
292
|
if (!quickDiagnosis?.lines.length && upload.diagnosis?.lines.length) {
|
|
164
|
-
console.log(
|
|
165
|
-
console.log(divider());
|
|
293
|
+
console.log(stylePrimary("Sentinel diagnosis"));
|
|
294
|
+
console.log(styleSecondary(divider()));
|
|
166
295
|
console.log("");
|
|
167
296
|
for (const line of upload.diagnosis.lines) {
|
|
168
|
-
console.log(line);
|
|
297
|
+
console.log(formatCliLine(line));
|
|
169
298
|
}
|
|
170
299
|
console.log("");
|
|
171
|
-
console.log(divider());
|
|
300
|
+
console.log(styleSecondary(divider()));
|
|
172
301
|
}
|
|
173
302
|
console.log("");
|
|
174
|
-
console.log("Debug report ready");
|
|
175
|
-
console.log(`
|
|
303
|
+
console.log(styleAction("Debug report ready"));
|
|
304
|
+
console.log(formatCliLine(`Next: ${upload.shareRunUrl || upload.internalRunUrl}`));
|
|
176
305
|
if (upload.shareLabel) {
|
|
177
|
-
console.log(` ${
|
|
306
|
+
console.log(` ${styleSecondary(upload.shareLabel)}`);
|
|
178
307
|
}
|
|
179
308
|
}
|
|
180
309
|
}
|
package/dist/runHistory.d.ts
CHANGED
|
@@ -12,6 +12,9 @@ export type FailedRunHistorySummary = {
|
|
|
12
12
|
stillFailing: number;
|
|
13
13
|
recurringCount: number;
|
|
14
14
|
recurringTitle: string | null;
|
|
15
|
+
dominantRecurringIssueTitle: string | null;
|
|
16
|
+
dominantRecurringIssueCount: number;
|
|
17
|
+
isDominantRecurringIssue: boolean;
|
|
15
18
|
};
|
|
16
19
|
export declare const buildRunDiffSummary: (playwrightJsonPath: string) => RunDiffSummary | null;
|
|
17
20
|
export declare const buildFailedRunHistorySummary: (playwrightJsonPath: string) => FailedRunHistorySummary | null;
|
package/dist/runHistory.js
CHANGED
|
@@ -204,6 +204,10 @@ const buildFailedRunHistorySummary = (playwrightJsonPath) => {
|
|
|
204
204
|
}))
|
|
205
205
|
.sort((a, b) => b.occurrences - a.occurrences);
|
|
206
206
|
const topRecurring = recurringFailures.find((item) => item.occurrences > 0) || null;
|
|
207
|
+
const topRecurringTotal = topRecurring ? topRecurring.occurrences + 1 : 0;
|
|
208
|
+
const isDominantRecurringIssue = Boolean(topRecurring &&
|
|
209
|
+
topRecurringTotal >= 3 &&
|
|
210
|
+
topRecurringTotal >= Math.ceil(Math.max(failingReporterRuns.length, 1) / 2));
|
|
207
211
|
writeSnapshot(snapshot);
|
|
208
212
|
const lines = [];
|
|
209
213
|
if (passStreakBeforeFailure > 0) {
|
|
@@ -220,6 +224,9 @@ const buildFailedRunHistorySummary = (playwrightJsonPath) => {
|
|
|
220
224
|
}
|
|
221
225
|
if (topRecurring) {
|
|
222
226
|
lines.push(`- Recurring across ${topRecurring.occurrences + 1} recorded failed runs in local history (${topRecurring.failure.title})`);
|
|
227
|
+
if (isDominantRecurringIssue) {
|
|
228
|
+
lines.push(`- This is the most common recent failure in local history`);
|
|
229
|
+
}
|
|
223
230
|
}
|
|
224
231
|
return lines.length
|
|
225
232
|
? {
|
|
@@ -230,7 +237,10 @@ const buildFailedRunHistorySummary = (playwrightJsonPath) => {
|
|
|
230
237
|
fixedTests,
|
|
231
238
|
stillFailing,
|
|
232
239
|
recurringCount: topRecurring ? topRecurring.occurrences : 0,
|
|
233
|
-
recurringTitle: topRecurring?.failure.title || null
|
|
240
|
+
recurringTitle: topRecurring?.failure.title || null,
|
|
241
|
+
dominantRecurringIssueTitle: topRecurring?.failure.title || null,
|
|
242
|
+
dominantRecurringIssueCount: topRecurringTotal,
|
|
243
|
+
isDominantRecurringIssue
|
|
234
244
|
}
|
|
235
245
|
: null;
|
|
236
246
|
};
|
package/dist/terminalSummary.js
CHANGED
|
@@ -73,6 +73,56 @@ const medianAbsoluteDeviation = (values) => {
|
|
|
73
73
|
return null;
|
|
74
74
|
return median(values.map((value) => Math.abs(value - med)));
|
|
75
75
|
};
|
|
76
|
+
const getPrimaryRiskKind = (candidate) => {
|
|
77
|
+
if (candidate.primaryReason.startsWith("failed in "))
|
|
78
|
+
return "historical_failures";
|
|
79
|
+
if (candidate.primaryReason.startsWith("passed after "))
|
|
80
|
+
return "retry";
|
|
81
|
+
if (candidate.primaryReason.startsWith("needed retries in "))
|
|
82
|
+
return "historical_retries";
|
|
83
|
+
if (candidate.primaryReason.startsWith("took "))
|
|
84
|
+
return "slowdown";
|
|
85
|
+
if (candidate.primaryReason.startsWith("has been unusually slow in "))
|
|
86
|
+
return "repeated_slow";
|
|
87
|
+
if (candidate.primaryReason.startsWith("used "))
|
|
88
|
+
return "timeout_pressure";
|
|
89
|
+
if (candidate.primaryReason.startsWith("is trending slower"))
|
|
90
|
+
return "trend";
|
|
91
|
+
if (candidate.retries > 0)
|
|
92
|
+
return "retry";
|
|
93
|
+
if (candidate.historicalRetries >= 2)
|
|
94
|
+
return "historical_retries";
|
|
95
|
+
if (candidate.ratio !== null && candidate.ratio >= 1.8)
|
|
96
|
+
return "slowdown";
|
|
97
|
+
if (candidate.repeatedSlowPasses >= 3)
|
|
98
|
+
return "repeated_slow";
|
|
99
|
+
if (candidate.recentTrendRatio !== null && candidate.recentTrendRatio >= 1.5)
|
|
100
|
+
return "trend";
|
|
101
|
+
if (candidate.timeoutUtilization !== null && candidate.timeoutUtilization >= 0.85)
|
|
102
|
+
return "timeout_pressure";
|
|
103
|
+
if (candidate.historicalFailures >= 5)
|
|
104
|
+
return "historical_failures";
|
|
105
|
+
return "generic";
|
|
106
|
+
};
|
|
107
|
+
const isStrongPassRisk = (candidate, lastFailureRunsAgo) => {
|
|
108
|
+
if (candidate.retries > 0)
|
|
109
|
+
return true;
|
|
110
|
+
if (candidate.ratio !== null && candidate.ratio >= 2.0)
|
|
111
|
+
return true;
|
|
112
|
+
if (candidate.repeatedSlowPasses >= 3)
|
|
113
|
+
return true;
|
|
114
|
+
if (candidate.timeoutUtilization !== null && candidate.timeoutUtilization >= 0.9)
|
|
115
|
+
return true;
|
|
116
|
+
if (candidate.historicalRetries >= 2)
|
|
117
|
+
return true;
|
|
118
|
+
if (candidate.historicalFailures >= 6 &&
|
|
119
|
+
lastFailureRunsAgo !== null &&
|
|
120
|
+
lastFailureRunsAgo <= 2 &&
|
|
121
|
+
(candidate.ratio !== null || candidate.historicalRetries > 0 || candidate.repeatedSlowPasses > 0)) {
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
return false;
|
|
125
|
+
};
|
|
76
126
|
const cleanTitlePath = (parts) => {
|
|
77
127
|
const normalized = parts.map((part) => part.trim()).filter(Boolean);
|
|
78
128
|
const withoutUnnamed = normalized.filter((part) => part !== "Unnamed test");
|
|
@@ -297,17 +347,33 @@ const buildPassingRunSummary = (playwrightJsonPath, options) => {
|
|
|
297
347
|
? `passed after ${test.retries} retr${test.retries === 1 ? 'y' : 'ies'} in this run`
|
|
298
348
|
: historicalRetries >= 2
|
|
299
349
|
? `needed retries in ${historicalRetries} of the last 10 passing runs`
|
|
300
|
-
:
|
|
301
|
-
? `
|
|
302
|
-
:
|
|
303
|
-
? `
|
|
304
|
-
:
|
|
305
|
-
? `
|
|
306
|
-
:
|
|
307
|
-
? `
|
|
308
|
-
:
|
|
309
|
-
? `
|
|
350
|
+
: ratio && ratio >= 1.8
|
|
351
|
+
? `took ${formatShortDuration(test.durationMs)} vs ${formatShortDuration(med)} recent median (${ratio.toFixed(1)}x)`
|
|
352
|
+
: repeatedSlowPasses >= 2
|
|
353
|
+
? `has been unusually slow in ${repeatedSlowPasses} of the last 10 passing runs`
|
|
354
|
+
: timeoutUtilization && timeoutUtilization >= 0.85
|
|
355
|
+
? `used ${(timeoutUtilization * 100).toFixed(0)}% of its timeout budget`
|
|
356
|
+
: recentTrendRatio && recentTrendRatio >= 1.5
|
|
357
|
+
? `is trending slower over recent runs`
|
|
358
|
+
: recentFailurePressure
|
|
359
|
+
? `failed in ${historicalFailures} of the last 10 runs`
|
|
310
360
|
: `has weak instability signals`;
|
|
361
|
+
const riskKind = getPrimaryRiskKind({
|
|
362
|
+
title: test.title,
|
|
363
|
+
ratio,
|
|
364
|
+
recentTrendRatio,
|
|
365
|
+
retries: test.retries,
|
|
366
|
+
historicalRetries,
|
|
367
|
+
historicalFailures,
|
|
368
|
+
repeatedSlowPasses,
|
|
369
|
+
timeoutUtilization,
|
|
370
|
+
medianDurationMs: med,
|
|
371
|
+
currentDurationMs: test.durationMs,
|
|
372
|
+
score,
|
|
373
|
+
strongSignal,
|
|
374
|
+
currentRunSignal,
|
|
375
|
+
primaryReason
|
|
376
|
+
});
|
|
311
377
|
if (score > 0) {
|
|
312
378
|
nearFailureCandidates.push({
|
|
313
379
|
title: test.title,
|
|
@@ -323,7 +389,8 @@ const buildPassingRunSummary = (playwrightJsonPath, options) => {
|
|
|
323
389
|
score,
|
|
324
390
|
strongSignal,
|
|
325
391
|
currentRunSignal,
|
|
326
|
-
primaryReason
|
|
392
|
+
primaryReason,
|
|
393
|
+
riskKind
|
|
327
394
|
});
|
|
328
395
|
}
|
|
329
396
|
}
|
|
@@ -333,27 +400,48 @@ const buildPassingRunSummary = (playwrightJsonPath, options) => {
|
|
|
333
400
|
(right.timeoutUtilization || 0) - (left.timeoutUtilization || 0) ||
|
|
334
401
|
(right.ratio || 0) - (left.ratio || 0));
|
|
335
402
|
const strongNearFailures = nearFailureCandidates.filter((candidate) => candidate.strongSignal &&
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
candidate.score >= 3);
|
|
341
|
-
const topNearFailures = strongNearFailures.slice(0, 2);
|
|
342
|
-
const flakyLookingCount = strongNearFailures.length;
|
|
343
|
-
const hasActiveRisk = flakyLookingCount > 0;
|
|
403
|
+
candidate.score >= 3 &&
|
|
404
|
+
isStrongPassRisk(candidate, lastFailureRunsAgo));
|
|
405
|
+
const topRisk = strongNearFailures[0] || null;
|
|
406
|
+
const hasActiveRisk = Boolean(topRisk);
|
|
344
407
|
const totalDurationMs = snapshot.tests.reduce((sum, test) => sum + test.durationMs, 0);
|
|
345
408
|
const displayedRunDurationMs = options?.observedRunDurationMs || snapshot.wallDurationMs || totalDurationMs;
|
|
346
409
|
const lines = ["All tests passed", `${snapshot.passedCount} tests in ${formatDuration(displayedRunDurationMs)}`];
|
|
347
|
-
if (hasActiveRisk &&
|
|
348
|
-
|
|
349
|
-
lines.push(
|
|
350
|
-
|
|
351
|
-
|
|
410
|
+
if (hasActiveRisk && topRisk) {
|
|
411
|
+
const riskKind = topRisk.riskKind;
|
|
412
|
+
lines.push(`At risk: ${topRisk.title} ${topRisk.primaryReason}`);
|
|
413
|
+
if (riskKind === "retry" || riskKind === "historical_retries") {
|
|
414
|
+
lines.push("Why this matters: Similar retry patterns often turn into flaky failures");
|
|
415
|
+
}
|
|
416
|
+
else if (riskKind === "slowdown" || riskKind === "repeated_slow" || riskKind === "trend") {
|
|
417
|
+
lines.push("Why this matters: Performance regressions often lead to flaky failures");
|
|
418
|
+
}
|
|
419
|
+
else if (riskKind === "historical_failures") {
|
|
420
|
+
lines.push("Why this matters: This test has been failing repeatedly and is likely to regress again");
|
|
421
|
+
}
|
|
422
|
+
else if (riskKind === "timeout_pressure") {
|
|
423
|
+
lines.push("Why this matters: Tests that run close to their timeout budget often become flaky");
|
|
424
|
+
}
|
|
425
|
+
else {
|
|
426
|
+
lines.push("Why this matters: This instability pattern often turns into a future failure");
|
|
427
|
+
}
|
|
352
428
|
if (lastFailureRunsAgo !== null)
|
|
353
429
|
lines.push(`Last failure: ${lastFailureRunsAgo} runs ago`);
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
430
|
+
if (riskKind === "retry" || riskKind === "historical_retries") {
|
|
431
|
+
lines.push("Recommendation: inspect retry behavior or timing around this test");
|
|
432
|
+
}
|
|
433
|
+
else if (riskKind === "slowdown" || riskKind === "repeated_slow" || riskKind === "trend") {
|
|
434
|
+
lines.push("Recommendation: monitor the next runs or investigate the slowdown");
|
|
435
|
+
}
|
|
436
|
+
else if (riskKind === "historical_failures") {
|
|
437
|
+
lines.push("Recommendation: rerun this test locally and inspect the last failing behavior");
|
|
438
|
+
}
|
|
439
|
+
else if (riskKind === "timeout_pressure") {
|
|
440
|
+
lines.push("Recommendation: inspect timeout pressure or waiting logic in this test");
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
lines.push("Recommendation: monitor the next runs and inspect this test if the pattern repeats");
|
|
444
|
+
}
|
|
357
445
|
}
|
|
358
446
|
else {
|
|
359
447
|
lines.push("No anomalies detected");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sentinelqa/playwright-reporter",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.54",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Playwright reporter for CI debugging with optional Sentinel cloud dashboards",
|
|
6
6
|
"license": "MIT",
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"@playwright/test": ">=1.40.0"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@sentinelqa/uploader": "^0.1.
|
|
42
|
+
"@sentinelqa/uploader": "^0.1.38"
|
|
43
43
|
},
|
|
44
44
|
"devDependencies": {
|
|
45
45
|
"@types/node": "^20.19.32",
|