@sentinelqa/playwright-reporter 0.1.51 → 0.1.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -0
- package/dist/quickDiagnosis.js +319 -35
- package/dist/reporter.js +116 -22
- package/dist/terminalSummary.js +72 -16
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -40,6 +40,46 @@ Debugging Playwright failures usually means downloading traces, screenshots, and
|
|
|
40
40
|
|
|
41
41
|
Reporter uploads those artifacts into a single hosted Sentinel run page so you can open one link, inspect failures fast, and share that link with the rest of the team.
|
|
42
42
|
|
|
43
|
+
## CLI diagnosis
|
|
44
|
+
|
|
45
|
+
Sentinel is not just a report link.
|
|
46
|
+
|
|
47
|
+
On failed runs, it prints a compact terminal diagnosis that answers:
|
|
48
|
+
|
|
49
|
+
1. what broke
|
|
50
|
+
2. why it broke
|
|
51
|
+
3. where to look
|
|
52
|
+
4. what changed
|
|
53
|
+
5. what to do next
|
|
54
|
+
|
|
55
|
+
The goal is simple:
|
|
56
|
+
|
|
57
|
+
you should not need to open logs first.
|
|
58
|
+
|
|
59
|
+
Typical CLI output:
|
|
60
|
+
|
|
61
|
+
```text
|
|
62
|
+
Sentinel diagnosis
|
|
63
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
64
|
+
|
|
65
|
+
NEW FAILURE after 8 passing runs
|
|
66
|
+
|
|
67
|
+
What broke: 10 tests failed
|
|
68
|
+
Why: collapsed into 2 real issues
|
|
69
|
+
|
|
70
|
+
Issue 1: UI assertion mismatch (6 tests)
|
|
71
|
+
Why: getByTestId('metric-pass-rate') showed "82%" instead of "88%"
|
|
72
|
+
Where: app.spec.ts:43, app.spec.ts:69
|
|
73
|
+
Expected: 88%
|
|
74
|
+
Received: 82%
|
|
75
|
+
What changed: "update analytics cards"
|
|
76
|
+
Next: verify getByTestId('metric-pass-rate')
|
|
77
|
+
Impact: 6 tests failing with same root cause
|
|
78
|
+
Clears: fixing this likely clears 6 of 10 failures
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
On passed runs, Sentinel stays short and only prints a warning if there is a strong risk signal worth caring about.
|
|
82
|
+
|
|
43
83
|
## Why teams use the free version
|
|
44
84
|
|
|
45
85
|
- Drop one wrapper into `playwright.config.ts` and keep running `npx playwright test`
|
package/dist/quickDiagnosis.js
CHANGED
|
@@ -162,6 +162,23 @@ const extractFocusLineFromSnippet = (snippet) => {
|
|
|
162
162
|
.replace(/^\s*\d+\s*\|\s*/, "")
|
|
163
163
|
.trim() || null;
|
|
164
164
|
};
|
|
165
|
+
const extractFocusLineFromMessage = (message) => {
|
|
166
|
+
if (!message)
|
|
167
|
+
return null;
|
|
168
|
+
const lines = stripAnsi(message).split(/\r?\n/);
|
|
169
|
+
const marked = lines.find((line) => /^\s*>\s*\d+\s*\|/.test(line));
|
|
170
|
+
if (marked) {
|
|
171
|
+
return marked
|
|
172
|
+
.replace(/^\s*>\s*/, "")
|
|
173
|
+
.replace(/^\s*\d+\s*\|\s*/, "")
|
|
174
|
+
.trim() || null;
|
|
175
|
+
}
|
|
176
|
+
const expectLine = lines.find((line) => /\b(await\s+)?expect\(/.test(line));
|
|
177
|
+
if (expectLine)
|
|
178
|
+
return expectLine.trim();
|
|
179
|
+
const stepLine = lines.find((line) => /\b(getBy|locator\(|page\.)/.test(line));
|
|
180
|
+
return stepLine?.trim() || null;
|
|
181
|
+
};
|
|
165
182
|
const parseCommitLine = (line) => {
|
|
166
183
|
const [sha, author, message] = line.split("\u001f");
|
|
167
184
|
if (!sha)
|
|
@@ -208,12 +225,28 @@ const normalizeStatus = (status) => {
|
|
|
208
225
|
return "passed";
|
|
209
226
|
return "skipped";
|
|
210
227
|
};
|
|
228
|
+
const errorRichness = (error) => {
|
|
229
|
+
if (!error)
|
|
230
|
+
return -1;
|
|
231
|
+
const message = String(error.message || error.stack || error.value || "");
|
|
232
|
+
let score = message.length;
|
|
233
|
+
if (error.location?.file)
|
|
234
|
+
score += 200;
|
|
235
|
+
if (/locator\.|Call log:|intercepts pointer events|waiting for|getBy|at .*:\d+:\d+/i.test(message))
|
|
236
|
+
score += 400;
|
|
237
|
+
if (/Test timeout of \d+ms exceeded\.$/i.test(stripAnsi(message).trim()))
|
|
238
|
+
score -= 300;
|
|
239
|
+
return score;
|
|
240
|
+
};
|
|
241
|
+
const selectBestError = (result) => {
|
|
242
|
+
const candidates = [result.error, ...(result.errors || [])].filter(Boolean);
|
|
243
|
+
if (!candidates.length)
|
|
244
|
+
return null;
|
|
245
|
+
return candidates.sort((a, b) => errorRichness(b) - errorRichness(a))[0] || null;
|
|
246
|
+
};
|
|
211
247
|
const toMessage = (result) => {
|
|
212
|
-
const
|
|
213
|
-
|
|
214
|
-
return stripAnsi(String(direct));
|
|
215
|
-
const first = result.errors?.find(Boolean);
|
|
216
|
-
return first ? stripAnsi(String(first.message || first.stack || first.value || "")) : "";
|
|
248
|
+
const best = selectBestError(result);
|
|
249
|
+
return best ? stripAnsi(String(best.message || best.stack || best.value || "")) : "";
|
|
217
250
|
};
|
|
218
251
|
const classifySignal = (message) => {
|
|
219
252
|
const lower = message.toLowerCase();
|
|
@@ -339,6 +372,56 @@ const describeDomState = (failure) => {
|
|
|
339
372
|
}
|
|
340
373
|
return null;
|
|
341
374
|
};
|
|
375
|
+
const dominantValue = (values) => {
|
|
376
|
+
const counts = new Map();
|
|
377
|
+
for (const value of values) {
|
|
378
|
+
const normalized = (value || "").trim();
|
|
379
|
+
if (!normalized)
|
|
380
|
+
continue;
|
|
381
|
+
counts.set(normalized, (counts.get(normalized) || 0) + 1);
|
|
382
|
+
}
|
|
383
|
+
const sorted = Array.from(counts.entries()).sort((a, b) => b[1] - a[1]);
|
|
384
|
+
if (!sorted.length)
|
|
385
|
+
return null;
|
|
386
|
+
return sorted[0][0];
|
|
387
|
+
};
|
|
388
|
+
const dominantFocusLine = (failures) => dominantValue(failures.map((failure) => failure.codeContext?.focusLine || null));
|
|
389
|
+
const timeoutState = (failure) => {
|
|
390
|
+
const dom = failure.domCapture;
|
|
391
|
+
if (!dom)
|
|
392
|
+
return "unknown";
|
|
393
|
+
if (dom.targetFound === false || dom.matchedCount === 0)
|
|
394
|
+
return "missing";
|
|
395
|
+
if (dom.visible === false)
|
|
396
|
+
return "hidden";
|
|
397
|
+
if (dom.enabled === false)
|
|
398
|
+
return "disabled";
|
|
399
|
+
if (dom.targetFound === true && dom.visible === true)
|
|
400
|
+
return "present";
|
|
401
|
+
return "unknown";
|
|
402
|
+
};
|
|
403
|
+
const sharedTimeoutEvidence = (failures) => {
|
|
404
|
+
const timeoutFailures = failures.filter((failure) => failure.signal === "timeout");
|
|
405
|
+
return {
|
|
406
|
+
action: dominantValue(timeoutFailures.map((failure) => failure.codeContext?.action || null)),
|
|
407
|
+
locator: dominantValue(timeoutFailures.map((failure) => failure.locator || null)),
|
|
408
|
+
state: dominantValue(timeoutFailures.map((failure) => timeoutState(failure))) || "unknown"
|
|
409
|
+
};
|
|
410
|
+
};
|
|
411
|
+
const timeoutStateLabel = (state) => {
|
|
412
|
+
switch (state) {
|
|
413
|
+
case "missing":
|
|
414
|
+
return "locator never appeared";
|
|
415
|
+
case "hidden":
|
|
416
|
+
return "found but hidden";
|
|
417
|
+
case "disabled":
|
|
418
|
+
return "found but disabled";
|
|
419
|
+
case "present":
|
|
420
|
+
return "found and visible before timeout";
|
|
421
|
+
default:
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
};
|
|
342
425
|
const buildTouchedFileReason = (label, files) => `${label}: ${files.slice(0, 2).map((file) => basename(file)).join(", ")}`;
|
|
343
426
|
const commitTouchedFailure = (commit, failure) => {
|
|
344
427
|
const reasons = [];
|
|
@@ -455,8 +538,12 @@ const flattenFailedCases = (node, titlePath = []) => {
|
|
|
455
538
|
timeoutBudgetMs: typeof test.timeout === "number" ? test.timeout : null,
|
|
456
539
|
codeContext: loadCodeContext(finalResult),
|
|
457
540
|
domCapture: loadDomCapture(finalResult),
|
|
458
|
-
errorLocation: finalResult
|
|
459
|
-
|
|
541
|
+
errorLocation: selectBestError(finalResult)?.location ||
|
|
542
|
+
finalResult.error?.location ||
|
|
543
|
+
finalResult.errors?.find((item) => item?.location)?.location ||
|
|
544
|
+
test.location ||
|
|
545
|
+
null,
|
|
546
|
+
errorSnippet: selectBestError(finalResult)?.snippet || finalResult.error?.snippet || null
|
|
460
547
|
}));
|
|
461
548
|
}
|
|
462
549
|
return failures;
|
|
@@ -546,13 +633,13 @@ const rootCauseLabel = (failure) => {
|
|
|
546
633
|
};
|
|
547
634
|
const describeFailure = (failure) => {
|
|
548
635
|
if (failure.signal === "assertion_mismatch" && failure.locator && failure.expected && failure.received) {
|
|
549
|
-
return `${failure.locator} showed "${failure.received}" instead of "${failure.expected}".`;
|
|
636
|
+
return `${compactLocator(failure.locator)} showed "${truncateValue(failure.received, 72)}" instead of "${truncateValue(failure.expected, 40)}".`;
|
|
550
637
|
}
|
|
551
638
|
if (failure.signal === "locator_not_found" && failure.locator) {
|
|
552
|
-
return `${failure.locator} was not found when the test expected it to be available.`;
|
|
639
|
+
return `${compactLocator(failure.locator)} was not found when the test expected it to be available.`;
|
|
553
640
|
}
|
|
554
641
|
if (failure.signal === "actionability" && failure.locator) {
|
|
555
|
-
return `${failure.locator} was found but was not actionable when the interaction ran.`;
|
|
642
|
+
return `${compactLocator(failure.locator)} was found but was not actionable when the interaction ran.`;
|
|
556
643
|
}
|
|
557
644
|
if (failure.signal === "network") {
|
|
558
645
|
return failure.apiHint
|
|
@@ -593,7 +680,25 @@ const describeCluster = (cluster) => {
|
|
|
593
680
|
return `${cluster.count} tests failed behind the same network or API signal.`;
|
|
594
681
|
}
|
|
595
682
|
if (cluster.count > 1 && failure.signal === "timeout") {
|
|
596
|
-
|
|
683
|
+
const evidence = sharedTimeoutEvidence(cluster.failures);
|
|
684
|
+
const action = evidence.action ? evidence.action.trim() : "the expected interaction";
|
|
685
|
+
const locator = evidence.locator ? compactLocator(evidence.locator) : "the target element";
|
|
686
|
+
if (evidence.state === "missing") {
|
|
687
|
+
return `${cluster.count} tests timed out because ${locator} never appeared before ${action}.`;
|
|
688
|
+
}
|
|
689
|
+
if (evidence.state === "hidden") {
|
|
690
|
+
return `${cluster.count} tests timed out because ${locator} stayed hidden and blocked ${action}.`;
|
|
691
|
+
}
|
|
692
|
+
if (evidence.state === "disabled") {
|
|
693
|
+
return `${cluster.count} tests timed out because ${locator} stayed disabled and blocked ${action}.`;
|
|
694
|
+
}
|
|
695
|
+
if (evidence.state === "present" && evidence.action) {
|
|
696
|
+
return `${cluster.count} tests timed out because ${action} on ${locator} never completed even though the target was present.`;
|
|
697
|
+
}
|
|
698
|
+
if (evidence.action && evidence.locator) {
|
|
699
|
+
return `${cluster.count} tests timed out on the same ${action} step for ${locator}.`;
|
|
700
|
+
}
|
|
701
|
+
return `${cluster.count} tests timed out while waiting for the same UI state change to complete.`;
|
|
597
702
|
}
|
|
598
703
|
if (cluster.count > 1 && failure.signal === "infra") {
|
|
599
704
|
return `${cluster.count} tests failed behind the same browser or CI instability signal.`;
|
|
@@ -610,8 +715,57 @@ const clusterCheckFirst = (cluster) => {
|
|
|
610
715
|
}
|
|
611
716
|
return "inspect the shared retry/flaky helper or intentional throw in these tests before opening the trace";
|
|
612
717
|
}
|
|
718
|
+
if (failure.signal === "timeout") {
|
|
719
|
+
const evidence = sharedTimeoutEvidence(cluster.failures);
|
|
720
|
+
const locator = evidence.locator ? compactLocator(evidence.locator) : "the target element";
|
|
721
|
+
if (evidence.state === "missing") {
|
|
722
|
+
return `make ${locator} render before the blocked step`;
|
|
723
|
+
}
|
|
724
|
+
if (evidence.state === "hidden") {
|
|
725
|
+
return `remove the condition keeping ${locator} hidden before the action`;
|
|
726
|
+
}
|
|
727
|
+
if (evidence.state === "disabled") {
|
|
728
|
+
return `enable ${locator} before the action runs`;
|
|
729
|
+
}
|
|
730
|
+
if (evidence.state === "present" && evidence.action && evidence.locator) {
|
|
731
|
+
return `fix what blocks ${evidence.action} on ${compactLocator(evidence.locator)} (overlay or pointer interception is likely)`;
|
|
732
|
+
}
|
|
733
|
+
if (failure.codeContext?.focusLine) {
|
|
734
|
+
return `inspect the waiting assertion or step: ${failure.codeContext.focusLine.trim()}`;
|
|
735
|
+
}
|
|
736
|
+
if (failure.likelyFile) {
|
|
737
|
+
return `inspect the waiting assertion or blocked state transition in ${basename(failure.likelyFile)}`;
|
|
738
|
+
}
|
|
739
|
+
}
|
|
613
740
|
return checkFirst(failure);
|
|
614
741
|
};
|
|
742
|
+
const buildClusterEvidenceLines = (cluster) => {
|
|
743
|
+
const failure = cluster.sample;
|
|
744
|
+
const lines = [];
|
|
745
|
+
if (failure.signal === "timeout") {
|
|
746
|
+
const evidence = sharedTimeoutEvidence(cluster.failures);
|
|
747
|
+
const focusLine = dominantFocusLine(cluster.failures);
|
|
748
|
+
if (focusLine)
|
|
749
|
+
lines.push(`Failing code: ${focusLine.trim()}`);
|
|
750
|
+
if (evidence.action)
|
|
751
|
+
lines.push(`Failing step: ${evidence.action}`);
|
|
752
|
+
if (evidence.locator)
|
|
753
|
+
lines.push(`Selector: ${compactLocator(evidence.locator)}`);
|
|
754
|
+
const stateLabel = timeoutStateLabel(evidence.state);
|
|
755
|
+
if (stateLabel)
|
|
756
|
+
lines.push(`Target state: ${stateLabel}`);
|
|
757
|
+
return lines.slice(0, 4);
|
|
758
|
+
}
|
|
759
|
+
const focusLine = dominantFocusLine(cluster.failures);
|
|
760
|
+
if (focusLine)
|
|
761
|
+
lines.push(`Failing code: ${focusLine.trim()}`);
|
|
762
|
+
for (const line of buildSecondaryEvidenceLines(failure)) {
|
|
763
|
+
if (lines.includes(line))
|
|
764
|
+
continue;
|
|
765
|
+
lines.push(line);
|
|
766
|
+
}
|
|
767
|
+
return lines.slice(0, 4);
|
|
768
|
+
};
|
|
615
769
|
const formatAffectedTests = (titles) => {
|
|
616
770
|
const unique = Array.from(new Set(titles.map((title) => shortenTitle(title)))).slice(0, 3);
|
|
617
771
|
if (!unique.length)
|
|
@@ -674,6 +828,18 @@ const buildEvidenceLines = (failure) => {
|
|
|
674
828
|
return lines.slice(0, 4);
|
|
675
829
|
};
|
|
676
830
|
const withoutPrefix = (value, prefix) => value.startsWith(prefix) ? value.slice(prefix.length).trim() : value;
|
|
831
|
+
const truncateValue = (value, max = 96) => {
|
|
832
|
+
if (!value)
|
|
833
|
+
return null;
|
|
834
|
+
const compact = value.replace(/\s+/g, " ").trim();
|
|
835
|
+
if (compact.length <= max)
|
|
836
|
+
return compact;
|
|
837
|
+
return `${compact.slice(0, max - 1)}…`;
|
|
838
|
+
};
|
|
839
|
+
const compactLocator = (value) => {
|
|
840
|
+
const compact = truncateValue(value, 40);
|
|
841
|
+
return compact || "target element";
|
|
842
|
+
};
|
|
677
843
|
const buildSecondaryEvidenceLines = (failure) => buildEvidenceLines(failure)
|
|
678
844
|
.filter((line) => !line.startsWith("Error location:") && !line.startsWith("Likely file:"))
|
|
679
845
|
.slice(0, 3);
|
|
@@ -704,6 +870,65 @@ const compactRootCauseSummary = (cluster) => {
|
|
|
704
870
|
}
|
|
705
871
|
return describeCluster(cluster);
|
|
706
872
|
};
|
|
873
|
+
const compactIssueTitle = (cluster) => {
|
|
874
|
+
const failure = cluster.sample;
|
|
875
|
+
const locator = compactLocator(failure.locator);
|
|
876
|
+
switch (failure.signal) {
|
|
877
|
+
case "assertion_mismatch":
|
|
878
|
+
return failure.locator ? `Assertion mismatch (${locator})` : "Assertion mismatch";
|
|
879
|
+
case "locator_not_found":
|
|
880
|
+
return failure.locator ? `Missing locator (${locator})` : "Missing locator";
|
|
881
|
+
case "actionability":
|
|
882
|
+
return failure.locator ? `Blocked interaction (${locator})` : "Blocked interaction";
|
|
883
|
+
case "network":
|
|
884
|
+
return failure.apiHint ? `Network/API failure (${truncateValue(failure.apiHint, 28)})` : "Network/API failure";
|
|
885
|
+
case "timeout":
|
|
886
|
+
return failure.locator ? `Timeout waiting on ${locator}` : "Timeout waiting for state change";
|
|
887
|
+
case "runtime":
|
|
888
|
+
return /retry|flaky/i.test(failure.message) ? "Test-side throw" : "Runtime error";
|
|
889
|
+
case "infra":
|
|
890
|
+
return "Browser/CI instability";
|
|
891
|
+
default:
|
|
892
|
+
return rootCauseLabel(failure);
|
|
893
|
+
}
|
|
894
|
+
};
|
|
895
|
+
const clusterCauseLine = (cluster) => {
|
|
896
|
+
const failure = cluster.sample;
|
|
897
|
+
if (failure.signal === "assertion_mismatch") {
|
|
898
|
+
return failure.locator
|
|
899
|
+
? `Same assertion mismatch on ${compactLocator(failure.locator)}`
|
|
900
|
+
: "Same assertion mismatch across these tests";
|
|
901
|
+
}
|
|
902
|
+
if (failure.signal === "locator_not_found") {
|
|
903
|
+
return failure.locator
|
|
904
|
+
? `${compactLocator(failure.locator)} is missing or changed in each failure`
|
|
905
|
+
: "Same missing or changed locator across these tests";
|
|
906
|
+
}
|
|
907
|
+
if (failure.signal === "actionability") {
|
|
908
|
+
return failure.locator
|
|
909
|
+
? `${compactLocator(failure.locator)} is present but blocked in each failure`
|
|
910
|
+
: "Same actionability problem across these tests";
|
|
911
|
+
}
|
|
912
|
+
return compactRootCauseSummary(cluster);
|
|
913
|
+
};
|
|
914
|
+
const strongerClusterNext = (cluster) => {
|
|
915
|
+
const failure = cluster.sample;
|
|
916
|
+
if (failure.signal === "assertion_mismatch") {
|
|
917
|
+
if (failure.locator && failure.apiHint) {
|
|
918
|
+
return `check ${compactLocator(failure.locator)} or the data returned by ${truncateValue(failure.apiHint, 36)}`;
|
|
919
|
+
}
|
|
920
|
+
if (failure.locator) {
|
|
921
|
+
return `check ${compactLocator(failure.locator)} or the data source behind it`;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
if (failure.signal === "locator_not_found" && failure.locator) {
|
|
925
|
+
return `check whether ${compactLocator(failure.locator)} changed or no longer renders`;
|
|
926
|
+
}
|
|
927
|
+
if (failure.signal === "actionability" && failure.locator) {
|
|
928
|
+
return `check what blocks ${compactLocator(failure.locator)} from becoming actionable`;
|
|
929
|
+
}
|
|
930
|
+
return clusterCheckFirst(cluster);
|
|
931
|
+
};
|
|
707
932
|
const compactErrorLine = (failure) => {
|
|
708
933
|
if (!failure.firstErrorLine)
|
|
709
934
|
return null;
|
|
@@ -736,10 +961,29 @@ const parseFailureFacts = (title, titlePath, message, status, file = null, optio
|
|
|
736
961
|
file: options.codeContext.file || fallbackCodeContext?.file || null,
|
|
737
962
|
line: options.codeContext.line ?? fallbackCodeContext?.line ?? null,
|
|
738
963
|
column: options.codeContext.column ?? fallbackCodeContext?.column ?? null,
|
|
739
|
-
focusLine: options.codeContext.focusLine || fallbackCodeContext?.focusLine || null,
|
|
964
|
+
focusLine: options.codeContext.focusLine || fallbackCodeContext?.focusLine || extractFocusLineFromMessage(message) || null,
|
|
740
965
|
found: options.codeContext.found ?? fallbackCodeContext?.found ?? false
|
|
741
966
|
}
|
|
742
|
-
: fallbackCodeContext
|
|
967
|
+
: fallbackCodeContext
|
|
968
|
+
? {
|
|
969
|
+
...fallbackCodeContext,
|
|
970
|
+
focusLine: fallbackCodeContext.focusLine || extractFocusLineFromMessage(message) || null
|
|
971
|
+
}
|
|
972
|
+
: {
|
|
973
|
+
file: fallbackLocation?.file || file || null,
|
|
974
|
+
line: typeof fallbackLocation?.line === "number" ? fallbackLocation.line : null,
|
|
975
|
+
column: typeof fallbackLocation?.column === "number" ? fallbackLocation.column : null,
|
|
976
|
+
action: null,
|
|
977
|
+
locator: null,
|
|
978
|
+
expectedText: null,
|
|
979
|
+
timeoutMs: null,
|
|
980
|
+
apiCall: null,
|
|
981
|
+
assertion: null,
|
|
982
|
+
methodName: null,
|
|
983
|
+
focusLine: extractFocusLineFromMessage(message),
|
|
984
|
+
previousActionLine: null,
|
|
985
|
+
found: Boolean(fallbackLocation?.file || extractFocusLineFromMessage(message))
|
|
986
|
+
};
|
|
743
987
|
const domCapture = options?.domCapture || null;
|
|
744
988
|
const locator = extractLocator(message) || codeContext?.locator || domCapture?.locator || null;
|
|
745
989
|
const expected = extractExpected(message) || codeContext?.expectedText || domCapture?.expectedText || null;
|
|
@@ -805,6 +1049,27 @@ const buildSimilarityKey = (failure) => {
|
|
|
805
1049
|
locationKey
|
|
806
1050
|
].join('|');
|
|
807
1051
|
}
|
|
1052
|
+
if (failure.signal === "assertion_mismatch") {
|
|
1053
|
+
return [
|
|
1054
|
+
failure.signal,
|
|
1055
|
+
failure.locator || failure.likelyModule || basename(failure.likelyFile) || "unknown-target",
|
|
1056
|
+
basename(failure.likelyFile) || "unknown-file"
|
|
1057
|
+
].join("|");
|
|
1058
|
+
}
|
|
1059
|
+
if (failure.signal === "timeout" || failure.signal === "actionability" || failure.signal === "locator_not_found") {
|
|
1060
|
+
return [
|
|
1061
|
+
failure.signal,
|
|
1062
|
+
failure.locator || failure.codeContext?.action || failure.likelyModule || "unknown-target",
|
|
1063
|
+
basename(failure.likelyFile) || "unknown-file"
|
|
1064
|
+
].join("|");
|
|
1065
|
+
}
|
|
1066
|
+
if (failure.signal === "network") {
|
|
1067
|
+
return [
|
|
1068
|
+
failure.signal,
|
|
1069
|
+
failure.apiHint || failure.likelyModule || "unknown-api",
|
|
1070
|
+
basename(failure.likelyFile) || "unknown-file"
|
|
1071
|
+
].join("|");
|
|
1072
|
+
}
|
|
808
1073
|
if (failure.locator || failure.expected || failure.received || failure.apiHint) {
|
|
809
1074
|
return [
|
|
810
1075
|
failure.signal,
|
|
@@ -860,22 +1125,26 @@ const buildQuickDiagnosis = (playwrightJsonPath) => {
|
|
|
860
1125
|
const failed = failures[0];
|
|
861
1126
|
const suspects = commitWindow.trusted ? rankCommitsForFailure(failed, commitWindow) : [];
|
|
862
1127
|
const top = suspects[0];
|
|
863
|
-
const lines = [`
|
|
1128
|
+
const lines = [`What broke: ${shortenTitle(failed.title)}`, `Why: ${(0, exports.describeFailure)(failed)}`];
|
|
864
1129
|
const primaryLocation = buildLocationLine(failed);
|
|
865
1130
|
const confidence = top ? confidenceLabel(top.score).toLowerCase() : "medium";
|
|
866
|
-
|
|
1131
|
+
if (primaryLocation)
|
|
1132
|
+
lines.push(`Where: ${withoutPrefix(withoutPrefix(primaryLocation, "Error location:"), "Likely file:")}`);
|
|
1133
|
+
if (failed.codeContext?.action)
|
|
1134
|
+
lines.push(`Failing step: ${failed.codeContext.action}`);
|
|
1135
|
+
if (failed.expected)
|
|
1136
|
+
lines.push(`Expected: ${truncateValue(failed.expected)}`);
|
|
1137
|
+
if (failed.received)
|
|
1138
|
+
lines.push(`Received: ${truncateValue(failed.received)}`);
|
|
867
1139
|
if (top && top.score >= 0.62) {
|
|
868
|
-
lines.push(`
|
|
1140
|
+
lines.push(`What changed: "${top.commit.message}"`);
|
|
869
1141
|
if (top.reasons.length) {
|
|
870
1142
|
lines.push(`Reason: ${compactWhyLine(top)}.`);
|
|
871
1143
|
}
|
|
872
1144
|
}
|
|
873
|
-
lines.push(
|
|
1145
|
+
lines.push(`Confidence: ${confidence}`);
|
|
1146
|
+
lines.push("Next:");
|
|
874
1147
|
lines.push(`- ${checkFirst(failed)}`);
|
|
875
|
-
if (primaryLocation)
|
|
876
|
-
lines.push(`Where: ${withoutPrefix(withoutPrefix(primaryLocation, "Error location:"), "Likely file:")}`);
|
|
877
|
-
if (failed.codeContext?.action)
|
|
878
|
-
lines.push(`Failing step: ${failed.codeContext.action}`);
|
|
879
1148
|
return {
|
|
880
1149
|
lines,
|
|
881
1150
|
footer: []
|
|
@@ -894,14 +1163,17 @@ const buildQuickDiagnosis = (playwrightJsonPath) => {
|
|
|
894
1163
|
count: clusterFailures.length,
|
|
895
1164
|
sample: clusterFailures[0],
|
|
896
1165
|
titles: clusterFailures.map((item) => item.title),
|
|
897
|
-
suspects: commitWindow.trusted ? buildClusterSuspects(clusterFailures, commitWindow) : []
|
|
1166
|
+
suspects: commitWindow.trusted ? buildClusterSuspects(clusterFailures, commitWindow) : [],
|
|
1167
|
+
failures: clusterFailures
|
|
898
1168
|
}))
|
|
899
|
-
.sort((a, b) => b.count - a.count
|
|
1169
|
+
.sort((a, b) => b.count - a.count ||
|
|
1170
|
+
((b.suspects[0]?.score || 0) - (a.suspects[0]?.score || 0)) ||
|
|
1171
|
+
(b.sample.signal === "assertion_mismatch" ? 1 : 0) - (a.sample.signal === "assertion_mismatch" ? 1 : 0))
|
|
900
1172
|
.slice(0, 2);
|
|
901
1173
|
const topCluster = clusters[0];
|
|
902
1174
|
const lines = [
|
|
903
|
-
`${failures.length} tests failed
|
|
904
|
-
`Collapsed into ${clusters.length}
|
|
1175
|
+
`${failures.length} tests failed`,
|
|
1176
|
+
`Collapsed into ${clusters.length} real issue${clusters.length === 1 ? "" : "s"}`
|
|
905
1177
|
];
|
|
906
1178
|
for (const [index, cluster] of clusters.entries()) {
|
|
907
1179
|
const clusterFailures = clusterMap.get(cluster.key) || [cluster.sample];
|
|
@@ -910,20 +1182,32 @@ const buildQuickDiagnosis = (playwrightJsonPath) => {
|
|
|
910
1182
|
const locationValue = clusterLocation
|
|
911
1183
|
? withoutPrefix(clusterLocation, clusterLocation.startsWith("Error locations:") ? "Error locations:" : "Error location:")
|
|
912
1184
|
: null;
|
|
913
|
-
const rootCause = cluster.count === 1 ? (0, exports.describeFailure)(cluster.sample) :
|
|
914
|
-
lines.push(`
|
|
915
|
-
lines.push(`
|
|
1185
|
+
const rootCause = cluster.count === 1 ? (0, exports.describeFailure)(cluster.sample) : clusterCauseLine(cluster);
|
|
1186
|
+
lines.push(`Issue ${index + 1}: ${compactIssueTitle(cluster)} (${cluster.count} test${cluster.count === 1 ? "" : "s"})`);
|
|
1187
|
+
lines.push(` Cause: ${rootCause}`);
|
|
1188
|
+
if (locationValue) {
|
|
1189
|
+
lines.push(` Where: ${locationValue}`);
|
|
1190
|
+
}
|
|
1191
|
+
for (const evidenceLine of buildClusterEvidenceLines(cluster)) {
|
|
1192
|
+
lines.push(` ${evidenceLine}`);
|
|
1193
|
+
}
|
|
1194
|
+
if (cluster.sample.expected) {
|
|
1195
|
+
lines.push(` Expected: ${truncateValue(cluster.sample.expected)}`);
|
|
1196
|
+
}
|
|
1197
|
+
if (cluster.sample.received) {
|
|
1198
|
+
lines.push(` Received: ${truncateValue(cluster.sample.received)}`);
|
|
1199
|
+
}
|
|
916
1200
|
if (top && top.score >= 0.62) {
|
|
917
|
-
lines.push(`
|
|
1201
|
+
lines.push(` What changed: "${top.commit.message}"`);
|
|
918
1202
|
if (top.reasons.length) {
|
|
919
|
-
lines.push(`
|
|
1203
|
+
lines.push(` Reason: ${compactWhyLine(top)}.`);
|
|
920
1204
|
}
|
|
921
1205
|
}
|
|
922
|
-
lines.push(`
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1206
|
+
lines.push(` Next: ${strongerClusterNext(cluster)}.`);
|
|
1207
|
+
lines.push(` Impact: ${cluster.count} test${cluster.count === 1 ? "" : "s"} failing with ${cluster.count === 1 ? "this" : "same"} root cause`);
|
|
1208
|
+
lines.push(` Clears: fixing this likely clears ${cluster.count} of ${failures.length} failures`);
|
|
1209
|
+
if (index < clusters.length - 1)
|
|
1210
|
+
lines.push("");
|
|
927
1211
|
}
|
|
928
1212
|
return {
|
|
929
1213
|
lines,
|
package/dist/reporter.js
CHANGED
|
@@ -11,10 +11,104 @@ const colorize = (value, code) => {
|
|
|
11
11
|
return value;
|
|
12
12
|
return `\u001b[${code}m${value}\u001b[0m`;
|
|
13
13
|
};
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
14
|
+
const styleCritical = (value) => colorize(value, "1;31");
|
|
15
|
+
const styleWarning = (value) => colorize(value, "1;33");
|
|
16
|
+
const styleAction = (value) => colorize(value, "1;36");
|
|
17
|
+
const stylePrimary = (value) => colorize(value, "1;97");
|
|
18
|
+
const styleSecondary = (value) => colorize(value, "2");
|
|
17
19
|
const divider = () => "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━";
|
|
20
|
+
const LABEL_WIDTH = 10;
|
|
21
|
+
const renderRow = (label, value, valueStyle, indent = "") => {
|
|
22
|
+
const paddedLabel = `${label}:`.padEnd(LABEL_WIDTH);
|
|
23
|
+
const styledValue = valueStyle ? valueStyle(value) : value;
|
|
24
|
+
return `${indent}${stylePrimary(paddedLabel)} ${styledValue}`;
|
|
25
|
+
};
|
|
26
|
+
const highlightCounts = (value) => value
|
|
27
|
+
.replace(/\b(\d+\s+tests?\s+failed)\b/gi, (_, match) => styleCritical(match))
|
|
28
|
+
.replace(/\b(\d+\s+real\s+issues?)\b/gi, (_, match) => styleWarning(match))
|
|
29
|
+
.replace(/\b(\d+\s+tests?)\b/gi, (_, match) => styleWarning(match))
|
|
30
|
+
.replace(/\b(\d+\s+previous\s+runs?)\b/gi, (_, match) => styleSecondary(match))
|
|
31
|
+
.replace(/\b(\d+\s+passing\s+runs?)\b/gi, (_, match) => styleSecondary(match))
|
|
32
|
+
.replace(/\b(\d+\s+newly\s+failing)\b/gi, (_, match) => styleWarning(match))
|
|
33
|
+
.replace(/\b(\d+\s+tests?\s+still\s+failing)\b/gi, (_, match) => styleWarning(match));
|
|
34
|
+
const styleIssueTitle = (value) => value.replace(/\((\d+\s+tests?)\)/i, (_, match) => `(${styleWarning(match)})`);
|
|
35
|
+
const styleWhereValue = (value) => value.replace(/\b([A-Za-z0-9_.-]+\.[cm]?[jt]sx?:\d+(?::\d+)?)\b/g, (_, match) => stylePrimary(match));
|
|
36
|
+
const formatCliLine = (line) => {
|
|
37
|
+
if (!line.trim())
|
|
38
|
+
return line;
|
|
39
|
+
if (line === divider())
|
|
40
|
+
return styleSecondary(line);
|
|
41
|
+
if (/^NEW FAILURE/.test(line))
|
|
42
|
+
return styleCritical(line);
|
|
43
|
+
if (/^RECURRING FAILURE/.test(line)) {
|
|
44
|
+
return line.replace(/RECURRING FAILURE/, styleWarning("RECURRING FAILURE")).replace(/\(([^)]+)\)/, (_, inner) => `(${styleSecondary(inner)})`);
|
|
45
|
+
}
|
|
46
|
+
if (/^All tests passed$/.test(line))
|
|
47
|
+
return stylePrimary(line);
|
|
48
|
+
if (/^\d+\s+tests?\s+failed$/.test(line))
|
|
49
|
+
return styleCritical(line);
|
|
50
|
+
if (/^Collapsed into \d+\s+real issue/.test(line))
|
|
51
|
+
return highlightCounts(line);
|
|
52
|
+
if (/^\d+\s+tests?\s+in\s+/.test(line))
|
|
53
|
+
return line;
|
|
54
|
+
if (/^Issue \d+:/.test(line)) {
|
|
55
|
+
const [head, rest] = line.split(": ", 2);
|
|
56
|
+
return `${stylePrimary(head)}: ${styleIssueTitle(rest || "")}`;
|
|
57
|
+
}
|
|
58
|
+
const rowMatch = line.match(/^(\s*)(What broke|Why|Cause|Where|What changed|Next|Expected|Received|Confidence|Impact|At risk|Why this matters|Recommendation|Last failure|Status|Artifacts ready|Failing step|Failing code|Selector|Target state|Clears|Report):\s*(.*)$/);
|
|
59
|
+
if (!rowMatch)
|
|
60
|
+
return line;
|
|
61
|
+
const [, indent, label, rawValue] = rowMatch;
|
|
62
|
+
const value = highlightCounts(rawValue);
|
|
63
|
+
switch (label) {
|
|
64
|
+
case "What broke":
|
|
65
|
+
return renderRow(label, value, undefined, indent);
|
|
66
|
+
case "Why":
|
|
67
|
+
return renderRow(label, value, undefined, indent);
|
|
68
|
+
case "Cause":
|
|
69
|
+
return renderRow(label, value, undefined, indent);
|
|
70
|
+
case "Where":
|
|
71
|
+
return renderRow(label, rawValue, styleWhereValue, indent);
|
|
72
|
+
case "What changed":
|
|
73
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
74
|
+
case "Next":
|
|
75
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
76
|
+
case "Report":
|
|
77
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
78
|
+
case "Expected":
|
|
79
|
+
return renderRow(label, rawValue, stylePrimary, indent);
|
|
80
|
+
case "Received":
|
|
81
|
+
return renderRow(label, rawValue, styleCritical, indent);
|
|
82
|
+
case "Confidence":
|
|
83
|
+
return renderRow(label, rawValue, /high/i.test(rawValue) ? styleAction : /medium/i.test(rawValue) ? styleWarning : styleSecondary, indent);
|
|
84
|
+
case "Impact":
|
|
85
|
+
return renderRow(label, highlightCounts(rawValue), undefined, indent);
|
|
86
|
+
case "At risk":
|
|
87
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
88
|
+
case "Why this matters":
|
|
89
|
+
return renderRow(label, rawValue, undefined, indent);
|
|
90
|
+
case "Recommendation":
|
|
91
|
+
return renderRow(label, rawValue, styleAction, indent);
|
|
92
|
+
case "Last failure":
|
|
93
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
94
|
+
case "Status":
|
|
95
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
96
|
+
case "Artifacts ready":
|
|
97
|
+
return renderRow(label, rawValue, styleSecondary, indent);
|
|
98
|
+
case "Failing step":
|
|
99
|
+
return renderRow(label, rawValue, stylePrimary, indent);
|
|
100
|
+
case "Failing code":
|
|
101
|
+
return renderRow(label, rawValue, stylePrimary, indent);
|
|
102
|
+
case "Selector":
|
|
103
|
+
return renderRow(label, rawValue, stylePrimary, indent);
|
|
104
|
+
case "Target state":
|
|
105
|
+
return renderRow(label, rawValue, styleWarning, indent);
|
|
106
|
+
case "Clears":
|
|
107
|
+
return renderRow(label, highlightCounts(rawValue), styleWarning, indent);
|
|
108
|
+
default:
|
|
109
|
+
return line;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
18
112
|
const readFinalFailedCount = (playwrightJsonPath) => {
|
|
19
113
|
try {
|
|
20
114
|
const parsed = JSON.parse(require("node:fs").readFileSync(playwrightJsonPath, "utf8"));
|
|
@@ -94,11 +188,11 @@ class SentinelReporter {
|
|
|
94
188
|
}
|
|
95
189
|
console.log("");
|
|
96
190
|
if (passingSummary) {
|
|
97
|
-
console.log(
|
|
98
|
-
console.log(divider());
|
|
191
|
+
console.log(stylePrimary("Sentinel run summary"));
|
|
192
|
+
console.log(styleSecondary(divider()));
|
|
99
193
|
console.log("");
|
|
100
194
|
for (const line of passingSummary.lines) {
|
|
101
|
-
console.log(line);
|
|
195
|
+
console.log(formatCliLine(line));
|
|
102
196
|
}
|
|
103
197
|
console.log("");
|
|
104
198
|
console.log(divider());
|
|
@@ -117,33 +211,33 @@ class SentinelReporter {
|
|
|
117
211
|
return;
|
|
118
212
|
}
|
|
119
213
|
if (quickDiagnosis?.lines.length) {
|
|
120
|
-
console.log(
|
|
121
|
-
console.log(divider());
|
|
214
|
+
console.log(stylePrimary("Sentinel diagnosis"));
|
|
215
|
+
console.log(styleSecondary(divider()));
|
|
122
216
|
console.log("");
|
|
123
217
|
if (failedRunHistory?.passStreakBeforeFailure && failedRunHistory.passStreakBeforeFailure > 0) {
|
|
124
|
-
console.log(`NEW FAILURE after ${failedRunHistory.passStreakBeforeFailure} passing runs`);
|
|
218
|
+
console.log(formatCliLine(`NEW FAILURE after ${failedRunHistory.passStreakBeforeFailure} passing runs`));
|
|
125
219
|
console.log("");
|
|
126
220
|
}
|
|
127
221
|
else if ((failedRunHistory?.recurringCount || 0) > 0) {
|
|
128
|
-
console.log(`RECURRING FAILURE (${failedRunHistory?.recurringCount} previous runs)`);
|
|
222
|
+
console.log(formatCliLine(`RECURRING FAILURE (${failedRunHistory?.recurringCount} previous runs)`));
|
|
129
223
|
console.log("");
|
|
130
224
|
}
|
|
131
225
|
for (const line of quickDiagnosis.lines) {
|
|
132
|
-
console.log(line);
|
|
226
|
+
console.log(formatCliLine(line));
|
|
133
227
|
}
|
|
134
228
|
console.log("");
|
|
135
|
-
console.log(divider());
|
|
229
|
+
console.log(styleSecondary(divider()));
|
|
136
230
|
console.log("");
|
|
137
231
|
if (failedRunHistory?.newFailures && failedRunHistory.newFailures > 0) {
|
|
138
|
-
console.log(`Impact: ${failedRunHistory.newFailures} newly failing in this run`);
|
|
232
|
+
console.log(formatCliLine(`Impact: ${failedRunHistory.newFailures} newly failing in this run`));
|
|
139
233
|
console.log("");
|
|
140
234
|
}
|
|
141
235
|
else if (failedRunHistory?.stillFailing && failedRunHistory.stillFailing > 0) {
|
|
142
|
-
console.log(`Impact: ${failedRunHistory.stillFailing} tests still failing`);
|
|
236
|
+
console.log(formatCliLine(`Impact: ${failedRunHistory.stillFailing} tests still failing`));
|
|
143
237
|
console.log("");
|
|
144
238
|
}
|
|
145
239
|
}
|
|
146
|
-
console.log("Uploading debug report...");
|
|
240
|
+
console.log(styleSecondary("Uploading debug report..."));
|
|
147
241
|
console.log("");
|
|
148
242
|
const upload = (await (0, node_1.runSentinelUpload)({
|
|
149
243
|
playwrightJsonPath: this.options.playwrightJsonPath,
|
|
@@ -161,20 +255,20 @@ class SentinelReporter {
|
|
|
161
255
|
throw new Error(`Sentinel upload failed with exit code ${upload.exitCode}`);
|
|
162
256
|
}
|
|
163
257
|
if (!quickDiagnosis?.lines.length && upload.diagnosis?.lines.length) {
|
|
164
|
-
console.log(
|
|
165
|
-
console.log(divider());
|
|
258
|
+
console.log(stylePrimary("Sentinel diagnosis"));
|
|
259
|
+
console.log(styleSecondary(divider()));
|
|
166
260
|
console.log("");
|
|
167
261
|
for (const line of upload.diagnosis.lines) {
|
|
168
|
-
console.log(line);
|
|
262
|
+
console.log(formatCliLine(line));
|
|
169
263
|
}
|
|
170
264
|
console.log("");
|
|
171
|
-
console.log(divider());
|
|
265
|
+
console.log(styleSecondary(divider()));
|
|
172
266
|
}
|
|
173
267
|
console.log("");
|
|
174
|
-
console.log("Debug report ready");
|
|
175
|
-
console.log(`
|
|
268
|
+
console.log(styleAction("Debug report ready"));
|
|
269
|
+
console.log(formatCliLine(`Next: ${upload.shareRunUrl || upload.internalRunUrl}`));
|
|
176
270
|
if (upload.shareLabel) {
|
|
177
|
-
console.log(` ${
|
|
271
|
+
console.log(` ${styleSecondary(upload.shareLabel)}`);
|
|
178
272
|
}
|
|
179
273
|
}
|
|
180
274
|
}
|
package/dist/terminalSummary.js
CHANGED
|
@@ -73,6 +73,37 @@ const medianAbsoluteDeviation = (values) => {
|
|
|
73
73
|
return null;
|
|
74
74
|
return median(values.map((value) => Math.abs(value - med)));
|
|
75
75
|
};
|
|
76
|
+
const getPrimaryRiskKind = (candidate) => {
|
|
77
|
+
if (candidate.primaryReason.startsWith("failed in "))
|
|
78
|
+
return "historical_failures";
|
|
79
|
+
if (candidate.primaryReason.startsWith("passed after "))
|
|
80
|
+
return "retry";
|
|
81
|
+
if (candidate.primaryReason.startsWith("needed retries in "))
|
|
82
|
+
return "historical_retries";
|
|
83
|
+
if (candidate.primaryReason.startsWith("took "))
|
|
84
|
+
return "slowdown";
|
|
85
|
+
if (candidate.primaryReason.startsWith("has been unusually slow in "))
|
|
86
|
+
return "repeated_slow";
|
|
87
|
+
if (candidate.primaryReason.startsWith("used "))
|
|
88
|
+
return "timeout_pressure";
|
|
89
|
+
if (candidate.primaryReason.startsWith("is trending slower"))
|
|
90
|
+
return "trend";
|
|
91
|
+
if (candidate.retries > 0)
|
|
92
|
+
return "retry";
|
|
93
|
+
if (candidate.historicalRetries >= 2)
|
|
94
|
+
return "historical_retries";
|
|
95
|
+
if (candidate.ratio !== null && candidate.ratio >= 1.8)
|
|
96
|
+
return "slowdown";
|
|
97
|
+
if (candidate.repeatedSlowPasses >= 3)
|
|
98
|
+
return "repeated_slow";
|
|
99
|
+
if (candidate.recentTrendRatio !== null && candidate.recentTrendRatio >= 1.5)
|
|
100
|
+
return "trend";
|
|
101
|
+
if (candidate.timeoutUtilization !== null && candidate.timeoutUtilization >= 0.85)
|
|
102
|
+
return "timeout_pressure";
|
|
103
|
+
if (candidate.historicalFailures >= 5)
|
|
104
|
+
return "historical_failures";
|
|
105
|
+
return "generic";
|
|
106
|
+
};
|
|
76
107
|
const cleanTitlePath = (parts) => {
|
|
77
108
|
const normalized = parts.map((part) => part.trim()).filter(Boolean);
|
|
78
109
|
const withoutUnnamed = normalized.filter((part) => part !== "Unnamed test");
|
|
@@ -297,16 +328,16 @@ const buildPassingRunSummary = (playwrightJsonPath, options) => {
|
|
|
297
328
|
? `passed after ${test.retries} retr${test.retries === 1 ? 'y' : 'ies'} in this run`
|
|
298
329
|
: historicalRetries >= 2
|
|
299
330
|
? `needed retries in ${historicalRetries} of the last 10 passing runs`
|
|
300
|
-
:
|
|
301
|
-
? `
|
|
302
|
-
:
|
|
303
|
-
? `
|
|
304
|
-
:
|
|
305
|
-
? `
|
|
306
|
-
:
|
|
307
|
-
? `
|
|
308
|
-
:
|
|
309
|
-
? `
|
|
331
|
+
: ratio && ratio >= 1.8
|
|
332
|
+
? `took ${formatShortDuration(test.durationMs)} vs ${formatShortDuration(med)} recent median (${ratio.toFixed(1)}x)`
|
|
333
|
+
: repeatedSlowPasses >= 2
|
|
334
|
+
? `has been unusually slow in ${repeatedSlowPasses} of the last 10 passing runs`
|
|
335
|
+
: timeoutUtilization && timeoutUtilization >= 0.85
|
|
336
|
+
? `used ${(timeoutUtilization * 100).toFixed(0)}% of its timeout budget`
|
|
337
|
+
: recentTrendRatio && recentTrendRatio >= 1.5
|
|
338
|
+
? `is trending slower over recent runs`
|
|
339
|
+
: recentFailurePressure
|
|
340
|
+
? `failed in ${historicalFailures} of the last 10 runs`
|
|
310
341
|
: `has weak instability signals`;
|
|
311
342
|
if (score > 0) {
|
|
312
343
|
nearFailureCandidates.push({
|
|
@@ -345,15 +376,40 @@ const buildPassingRunSummary = (playwrightJsonPath, options) => {
|
|
|
345
376
|
const displayedRunDurationMs = options?.observedRunDurationMs || snapshot.wallDurationMs || totalDurationMs;
|
|
346
377
|
const lines = ["All tests passed", `${snapshot.passedCount} tests in ${formatDuration(displayedRunDurationMs)}`];
|
|
347
378
|
if (hasActiveRisk && topNearFailures[0]) {
|
|
379
|
+
const riskKind = getPrimaryRiskKind(topNearFailures[0]);
|
|
348
380
|
lines.push(`At risk: ${topNearFailures[0].title} ${topNearFailures[0].primaryReason}`);
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
381
|
+
if (riskKind === "retry" || riskKind === "historical_retries") {
|
|
382
|
+
lines.push("Why this matters: Similar retry patterns often turn into flaky failures");
|
|
383
|
+
}
|
|
384
|
+
else if (riskKind === "slowdown" || riskKind === "repeated_slow" || riskKind === "trend") {
|
|
385
|
+
lines.push("Why this matters: Performance regressions often lead to flaky failures");
|
|
386
|
+
}
|
|
387
|
+
else if (riskKind === "historical_failures") {
|
|
388
|
+
lines.push("Why this matters: This test has been failing repeatedly and is likely to regress again");
|
|
389
|
+
}
|
|
390
|
+
else if (riskKind === "timeout_pressure") {
|
|
391
|
+
lines.push("Why this matters: Tests that run close to their timeout budget often become flaky");
|
|
392
|
+
}
|
|
393
|
+
else {
|
|
394
|
+
lines.push("Why this matters: This instability pattern often turns into a future failure");
|
|
395
|
+
}
|
|
352
396
|
if (lastFailureRunsAgo !== null)
|
|
353
397
|
lines.push(`Last failure: ${lastFailureRunsAgo} runs ago`);
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
398
|
+
if (riskKind === "retry" || riskKind === "historical_retries") {
|
|
399
|
+
lines.push("Recommendation: inspect retry behavior or timing around this test");
|
|
400
|
+
}
|
|
401
|
+
else if (riskKind === "slowdown" || riskKind === "repeated_slow" || riskKind === "trend") {
|
|
402
|
+
lines.push("Recommendation: monitor the next runs or investigate the slowdown");
|
|
403
|
+
}
|
|
404
|
+
else if (riskKind === "historical_failures") {
|
|
405
|
+
lines.push("Recommendation: rerun this test locally and inspect the last failing behavior");
|
|
406
|
+
}
|
|
407
|
+
else if (riskKind === "timeout_pressure") {
|
|
408
|
+
lines.push("Recommendation: inspect timeout pressure or waiting logic in this test");
|
|
409
|
+
}
|
|
410
|
+
else {
|
|
411
|
+
lines.push("Recommendation: monitor the next runs and inspect this test if the pattern repeats");
|
|
412
|
+
}
|
|
357
413
|
}
|
|
358
414
|
else {
|
|
359
415
|
lines.push("No anomalies detected");
|