donobu 5.60.1 → 5.60.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/donobu-cli.js +151 -52
- package/dist/envVars.d.ts +4 -0
- package/dist/envVars.js +12 -0
- package/dist/esm/cli/donobu-cli.js +151 -52
- package/dist/esm/envVars.d.ts +4 -0
- package/dist/esm/envVars.js +12 -0
- package/dist/esm/lib/page/extendPage.d.ts +6 -0
- package/dist/esm/lib/page/extendPage.js +24 -1
- package/dist/esm/lib/test/healRerunGate.d.ts +85 -0
- package/dist/esm/lib/test/healRerunGate.js +186 -0
- package/dist/esm/lib/test/testExtension.d.ts +1 -0
- package/dist/esm/lib/test/testExtension.js +20 -10
- package/dist/esm/reporter/buildReport.js +32 -0
- package/dist/esm/reporter/merge.d.ts +1 -6
- package/dist/esm/reporter/merge.js +57 -35
- package/dist/esm/reporter/model.d.ts +9 -0
- package/dist/esm/reporter/model.js +10 -1
- package/dist/esm/reporter/render.js +34 -12
- package/dist/esm/reporter/renderMarkdown.js +148 -93
- package/dist/esm/reporter/renderSlack.js +39 -28
- package/dist/esm/reporter/reportWalk.d.ts +16 -6
- package/dist/esm/reporter/reportWalk.js +63 -13
- package/dist/lib/page/extendPage.d.ts +6 -0
- package/dist/lib/page/extendPage.js +24 -1
- package/dist/lib/test/healRerunGate.d.ts +85 -0
- package/dist/lib/test/healRerunGate.js +186 -0
- package/dist/lib/test/testExtension.d.ts +1 -0
- package/dist/lib/test/testExtension.js +20 -10
- package/dist/reporter/buildReport.js +32 -0
- package/dist/reporter/merge.d.ts +1 -6
- package/dist/reporter/merge.js +57 -35
- package/dist/reporter/model.d.ts +9 -0
- package/dist/reporter/model.js +10 -1
- package/dist/reporter/render.js +34 -12
- package/dist/reporter/renderMarkdown.js +148 -93
- package/dist/reporter/renderSlack.js +39 -28
- package/dist/reporter/reportWalk.d.ts +16 -6
- package/dist/reporter/reportWalk.js +63 -13
- package/package.json +1 -1
|
@@ -11,6 +11,34 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
11
11
|
exports.renderMarkdown = renderMarkdown;
|
|
12
12
|
const ansi_1 = require("../utils/ansi");
|
|
13
13
|
const reportWalk_1 = require("./reportWalk");
|
|
14
|
+
const STATUS_LABELS = {
|
|
15
|
+
passed: '✅ Passed',
|
|
16
|
+
flaky: '🔁 Flaky',
|
|
17
|
+
expectedFailure: '☑️ Expected Failure',
|
|
18
|
+
healed: '❤️🩹 Healed',
|
|
19
|
+
failed: '❌ Failed',
|
|
20
|
+
timedOut: '⏰ Timed Out',
|
|
21
|
+
skipped: '⏭️ Skipped',
|
|
22
|
+
interrupted: '⚡ Interrupted',
|
|
23
|
+
unknown: '⚠️ Unknown',
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* The attempt whose details (duration, errors) the per-test section shows.
|
|
27
|
+
* For surviving failures this is the last failing attempt — the final entry
|
|
28
|
+
* in `results` may be a heal-rerun skip that carries no error context.
|
|
29
|
+
*/
|
|
30
|
+
function displayResultOf(test, status) {
|
|
31
|
+
const results = test.results ?? [];
|
|
32
|
+
if (status === 'failed' ||
|
|
33
|
+
status === 'timedOut' ||
|
|
34
|
+
status === 'interrupted') {
|
|
35
|
+
const failures = results.filter((r) => ['failed', 'timedOut', 'interrupted'].includes(r?.status));
|
|
36
|
+
if (failures.length > 0) {
|
|
37
|
+
return failures.at(-1);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return results.at(-1);
|
|
41
|
+
}
|
|
14
42
|
function formatDuration(ms) {
|
|
15
43
|
if (ms < 1000) {
|
|
16
44
|
return `${ms}ms`;
|
|
@@ -29,69 +57,111 @@ function renderMarkdown(report) {
|
|
|
29
57
|
if (report.metadata?.donobuMergedReport) {
|
|
30
58
|
markdown += `> ⚙️ Auto-heal summary generated by Donobu (merged initial and retry runs).\n\n`;
|
|
31
59
|
}
|
|
32
|
-
// Per-file summary table
|
|
60
|
+
// Per-file summary table. Counts are computed for all suites first so the
|
|
61
|
+
// rarely-used Expected Failures column can be omitted when it would be
|
|
62
|
+
// entirely empty.
|
|
33
63
|
markdown += `## Summary\n\n`;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
selfHealed++;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
case 'interrupted':
|
|
78
|
-
interrupted++;
|
|
79
|
-
break;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
64
|
+
const suiteRows = suites.map((suite) => {
|
|
65
|
+
const row = {
|
|
66
|
+
file: suite.file,
|
|
67
|
+
passed: 0,
|
|
68
|
+
flaky: 0,
|
|
69
|
+
expectedFailures: 0,
|
|
70
|
+
selfHealed: 0,
|
|
71
|
+
failed: 0,
|
|
72
|
+
timedOut: 0,
|
|
73
|
+
skipped: 0,
|
|
74
|
+
interrupted: 0,
|
|
75
|
+
duration: 0,
|
|
76
|
+
};
|
|
77
|
+
for (const spec of (0, reportWalk_1.collectSpecs)(suite)) {
|
|
78
|
+
for (const test of spec.tests ?? []) {
|
|
79
|
+
row.duration += test.results?.at(-1)?.duration || 0;
|
|
80
|
+
switch ((0, reportWalk_1.statusOf)(test)) {
|
|
81
|
+
case 'passed':
|
|
82
|
+
row.passed++;
|
|
83
|
+
break;
|
|
84
|
+
case 'flaky':
|
|
85
|
+
row.flaky++;
|
|
86
|
+
break;
|
|
87
|
+
case 'expectedFailure':
|
|
88
|
+
row.expectedFailures++;
|
|
89
|
+
break;
|
|
90
|
+
case 'healed':
|
|
91
|
+
row.selfHealed++;
|
|
92
|
+
break;
|
|
93
|
+
case 'timedOut':
|
|
94
|
+
row.timedOut++;
|
|
95
|
+
break;
|
|
96
|
+
case 'skipped':
|
|
97
|
+
row.skipped++;
|
|
98
|
+
break;
|
|
99
|
+
case 'interrupted':
|
|
100
|
+
row.interrupted++;
|
|
101
|
+
break;
|
|
102
|
+
default:
|
|
103
|
+
// 'failed' and 'unknown' both belong in the Failed column —
|
|
104
|
+
// dropping unknowns from the table would hide real problems.
|
|
105
|
+
row.failed++;
|
|
106
|
+
break;
|
|
82
107
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
totalFailed += failed;
|
|
87
|
-
totalTimedOut += timedOut;
|
|
88
|
-
totalSkipped += skipped;
|
|
89
|
-
totalInterrupted += interrupted;
|
|
90
|
-
totalSelfHealed += selfHealed;
|
|
91
|
-
totalDuration += fileDuration;
|
|
92
|
-
markdown += `| ${suite.file} | ${passed ? passed + ' ✅' : ''} | ${selfHealed ? selfHealed + ' ❤️🩹' : ''} | ${failed ? failed + ' ❌' : ''} | ${timedOut ? timedOut + ' ⏰' : ''} | ${skipped ? skipped + ' ⏭️' : ''} | ${interrupted ? interrupted + ' ⚡' : ''} | ${formatDuration(fileDuration)} |\n`;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return row;
|
|
93
111
|
});
|
|
94
|
-
|
|
112
|
+
const totals = suiteRows.reduce((acc, row) => ({
|
|
113
|
+
file: '**TOTAL**',
|
|
114
|
+
passed: acc.passed + row.passed,
|
|
115
|
+
flaky: acc.flaky + row.flaky,
|
|
116
|
+
expectedFailures: acc.expectedFailures + row.expectedFailures,
|
|
117
|
+
selfHealed: acc.selfHealed + row.selfHealed,
|
|
118
|
+
failed: acc.failed + row.failed,
|
|
119
|
+
timedOut: acc.timedOut + row.timedOut,
|
|
120
|
+
skipped: acc.skipped + row.skipped,
|
|
121
|
+
interrupted: acc.interrupted + row.interrupted,
|
|
122
|
+
duration: acc.duration + row.duration,
|
|
123
|
+
}), {
|
|
124
|
+
file: '**TOTAL**',
|
|
125
|
+
passed: 0,
|
|
126
|
+
flaky: 0,
|
|
127
|
+
expectedFailures: 0,
|
|
128
|
+
selfHealed: 0,
|
|
129
|
+
failed: 0,
|
|
130
|
+
timedOut: 0,
|
|
131
|
+
skipped: 0,
|
|
132
|
+
interrupted: 0,
|
|
133
|
+
duration: 0,
|
|
134
|
+
});
|
|
135
|
+
const includeExpectedFailures = totals.expectedFailures > 0;
|
|
136
|
+
const expectedFailuresHeader = includeExpectedFailures
|
|
137
|
+
? ` Expected Failures |`
|
|
138
|
+
: '';
|
|
139
|
+
markdown += `| File | Passed | Flaky |${expectedFailuresHeader} Self-Healed | Failed | Timed Out | Skipped | Interrupted | Duration |\n`;
|
|
140
|
+
markdown += `| - | - | - |${includeExpectedFailures ? ' - |' : ''} - | - | - | - | - | - |\n`;
|
|
141
|
+
const cell = (count, emoji, bold) => {
|
|
142
|
+
const text = bold
|
|
143
|
+
? `**${count} ${emoji}**`
|
|
144
|
+
: count
|
|
145
|
+
? `${count} ${emoji}`
|
|
146
|
+
: '';
|
|
147
|
+
return ` ${text} |`;
|
|
148
|
+
};
|
|
149
|
+
for (const row of [...suiteRows, totals]) {
|
|
150
|
+
const bold = row === totals;
|
|
151
|
+
markdown += `| ${row.file} |`;
|
|
152
|
+
markdown += cell(row.passed, '✅', bold);
|
|
153
|
+
markdown += cell(row.flaky, '🔁', bold);
|
|
154
|
+
if (includeExpectedFailures) {
|
|
155
|
+
markdown += cell(row.expectedFailures, '☑️', bold);
|
|
156
|
+
}
|
|
157
|
+
markdown += cell(row.selfHealed, '❤️🩹', bold);
|
|
158
|
+
markdown += cell(row.failed, '❌', bold);
|
|
159
|
+
markdown += cell(row.timedOut, '⏰', bold);
|
|
160
|
+
markdown += cell(row.skipped, '⏭️', bold);
|
|
161
|
+
markdown += cell(row.interrupted, '⚡', bold);
|
|
162
|
+
const duration = formatDuration(row.duration);
|
|
163
|
+
markdown += ` ${bold ? `**${duration}**` : duration} |\n`;
|
|
164
|
+
}
|
|
95
165
|
markdown += `\n`;
|
|
96
166
|
// Per-test details
|
|
97
167
|
suites.forEach((suite) => {
|
|
@@ -100,9 +170,9 @@ function renderMarkdown(report) {
|
|
|
100
170
|
(0, reportWalk_1.collectSpecs)(suite).forEach((spec) => {
|
|
101
171
|
markdown += `### ${spec.title}\n\n`;
|
|
102
172
|
(spec.tests ?? []).forEach((test) => {
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
173
|
+
const status = (0, reportWalk_1.statusOf)(test);
|
|
174
|
+
const result = displayResultOf(test, status);
|
|
175
|
+
if (status === 'skipped') {
|
|
106
176
|
markdown += `**Status**: ⏭️ Skipped \n`;
|
|
107
177
|
markdown += `**Duration**: N/A \n`;
|
|
108
178
|
if (Array.isArray(test.tags) && test.tags.length > 0) {
|
|
@@ -116,47 +186,32 @@ function renderMarkdown(report) {
|
|
|
116
186
|
markdown += `\n---\n\n`;
|
|
117
187
|
return;
|
|
118
188
|
}
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
if (healed) {
|
|
122
|
-
status = '❤️🩹 Healed';
|
|
123
|
-
}
|
|
124
|
-
else {
|
|
125
|
-
switch (result.status) {
|
|
126
|
-
case 'passed':
|
|
127
|
-
status = '✅ Passed';
|
|
128
|
-
break;
|
|
129
|
-
case 'failed':
|
|
130
|
-
status = '❌ Failed';
|
|
131
|
-
break;
|
|
132
|
-
case 'timedOut':
|
|
133
|
-
status = '⏰ Timed Out';
|
|
134
|
-
break;
|
|
135
|
-
case 'skipped':
|
|
136
|
-
status = '⏭️ Skipped';
|
|
137
|
-
break;
|
|
138
|
-
case 'interrupted':
|
|
139
|
-
status = '⚡ Interrupted';
|
|
140
|
-
break;
|
|
141
|
-
default:
|
|
142
|
-
status = `⚠️ ${result.status || 'Unknown'}`;
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
const duration = formatDuration(result.duration || 0);
|
|
146
|
-
markdown += `**Status**: ${status} \n`;
|
|
189
|
+
const duration = formatDuration(result?.duration || 0);
|
|
190
|
+
markdown += `**Status**: ${STATUS_LABELS[status]} \n`;
|
|
147
191
|
markdown += `**Duration**: ${duration} \n`;
|
|
148
192
|
if (Array.isArray(test.tags) && test.tags.length > 0) {
|
|
149
193
|
markdown += `**Tags**: ${test.tags.join(' ')} \n`;
|
|
150
194
|
}
|
|
151
|
-
if (healed) {
|
|
195
|
+
if (status === 'healed') {
|
|
152
196
|
markdown += `> ❤️🩹 This test was automatically healed by re-running with Donobu treatment plan directives.\n\n`;
|
|
153
197
|
}
|
|
198
|
+
if (status === 'flaky') {
|
|
199
|
+
markdown += `> 🔁 This test failed and then passed on a retry.\n\n`;
|
|
200
|
+
}
|
|
201
|
+
const notReattempted = test.annotations?.find((a) => a.type === 'auto-heal-not-reattempted');
|
|
202
|
+
if (notReattempted) {
|
|
203
|
+
markdown += `> ⚠️ ${notReattempted.description}\n\n`;
|
|
204
|
+
}
|
|
205
|
+
const rerunFailed = test.annotations?.find((a) => a.type === 'auto-heal-rerun-failed');
|
|
206
|
+
if (rerunFailed) {
|
|
207
|
+
markdown += `> ⚠️ ${rerunFailed.description}\n\n`;
|
|
208
|
+
}
|
|
154
209
|
const objectiveAnnotation = test.annotations?.find((a) => a.type === 'objective');
|
|
155
210
|
if (objectiveAnnotation) {
|
|
156
211
|
const objective = (objectiveAnnotation.description || 'No objective provided').replace(/```/g, '\\`\\`\\`');
|
|
157
212
|
markdown += `**Objective**:\n\`\`\`\n${objective}\n\`\`\`\n`;
|
|
158
213
|
}
|
|
159
|
-
if (result
|
|
214
|
+
if (result?.status === 'failed' && result.error) {
|
|
160
215
|
markdown += `\n<details>\n<summary>⚠️ Error Details</summary>\n\n`;
|
|
161
216
|
markdown += `\`\`\`\n${(0, ansi_1.stripAnsi)(result.error.message || '') || 'No error message available'}\n\`\`\`\n\n`;
|
|
162
217
|
if (result.error.snippet) {
|
|
@@ -17,6 +17,8 @@ function renderSlack(report, options = {}) {
|
|
|
17
17
|
text: { type: 'plain_text', text: '🐵 Donobu Test Summary' },
|
|
18
18
|
});
|
|
19
19
|
let totalPassed = 0;
|
|
20
|
+
let totalFlaky = 0;
|
|
21
|
+
let totalExpectedFailures = 0;
|
|
20
22
|
let totalFailed = 0;
|
|
21
23
|
let totalTimedOut = 0;
|
|
22
24
|
let totalSkipped = 0;
|
|
@@ -25,41 +27,50 @@ function renderSlack(report, options = {}) {
|
|
|
25
27
|
suites.forEach((suite) => {
|
|
26
28
|
(0, reportWalk_1.collectSpecs)(suite).forEach((spec) => {
|
|
27
29
|
(spec.tests ?? []).forEach((test) => {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
30
|
+
switch ((0, reportWalk_1.statusOf)(test)) {
|
|
31
|
+
case 'passed':
|
|
32
|
+
totalPassed++;
|
|
33
|
+
break;
|
|
34
|
+
case 'flaky':
|
|
35
|
+
totalFlaky++;
|
|
36
|
+
break;
|
|
37
|
+
case 'expectedFailure':
|
|
38
|
+
totalExpectedFailures++;
|
|
39
|
+
break;
|
|
40
|
+
case 'healed':
|
|
36
41
|
totalSelfHealed++;
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
break;
|
|
52
|
-
case 'interrupted':
|
|
53
|
-
totalInterrupted++;
|
|
54
|
-
break;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
42
|
+
break;
|
|
43
|
+
case 'timedOut':
|
|
44
|
+
totalTimedOut++;
|
|
45
|
+
break;
|
|
46
|
+
case 'skipped':
|
|
47
|
+
totalSkipped++;
|
|
48
|
+
break;
|
|
49
|
+
case 'interrupted':
|
|
50
|
+
totalInterrupted++;
|
|
51
|
+
break;
|
|
52
|
+
default:
|
|
53
|
+
// 'failed' and 'unknown' both count as failures.
|
|
54
|
+
totalFailed++;
|
|
55
|
+
break;
|
|
57
56
|
}
|
|
58
57
|
});
|
|
59
58
|
});
|
|
60
59
|
});
|
|
61
60
|
const statusRows = [
|
|
62
61
|
{ name: 'Passed', emoji: '✅', count: totalPassed },
|
|
62
|
+
{ name: 'Flaky', emoji: '🔁', count: totalFlaky },
|
|
63
|
+
// Expected failures are rare enough that a permanent zero row would be
|
|
64
|
+
// clutter; anyone using test.fail() will see the row when it matters.
|
|
65
|
+
...(totalExpectedFailures > 0
|
|
66
|
+
? [
|
|
67
|
+
{
|
|
68
|
+
name: 'Expected Failures',
|
|
69
|
+
emoji: '☑️',
|
|
70
|
+
count: totalExpectedFailures,
|
|
71
|
+
},
|
|
72
|
+
]
|
|
73
|
+
: []),
|
|
63
74
|
{ name: 'Self-Healed', emoji: '❤️🩹', count: totalSelfHealed },
|
|
64
75
|
{ name: 'Failed', emoji: '❌', count: totalFailed },
|
|
65
76
|
{ name: 'Timed Out', emoji: '⏰', count: totalTimedOut },
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Shared helpers for walking a `DonobuReport` and classifying
|
|
3
|
-
* the tests inside it. Used by every renderer (HTML, Markdown, Slack)
|
|
4
|
-
* all agree on "what counts as self-healed"
|
|
3
|
+
* the tests inside it. Used by every renderer (HTML, Markdown, Slack) and the
|
|
4
|
+
* merge step's stats so they all agree on "what counts as self-healed",
|
|
5
|
+
* "what counts as flaky", and how suites nest.
|
|
5
6
|
*/
|
|
6
7
|
/**
|
|
7
8
|
* Recursively collect all specs from a suite and its nested sub-suites. The
|
|
@@ -18,11 +19,20 @@ export declare function collectSpecs(suite: any): any[];
|
|
|
18
19
|
*/
|
|
19
20
|
export declare function isSelfHealed(test: any): boolean;
|
|
20
21
|
/** Normalized test-status the renderers use for counts and labels. */
|
|
21
|
-
export type NormalizedStatus = 'passed' | 'failed' | 'healed' | 'timedOut' | 'skipped' | 'interrupted' | 'unknown';
|
|
22
|
+
export type NormalizedStatus = 'passed' | 'flaky' | 'expectedFailure' | 'failed' | 'healed' | 'timedOut' | 'skipped' | 'interrupted' | 'unknown';
|
|
22
23
|
/**
|
|
23
|
-
* Derive the display status for a test
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* Derive the display status for a test from its full attempt history, not
|
|
25
|
+
* just the final attempt. The distinctions that matter:
|
|
26
|
+
*
|
|
27
|
+
* - A trailing `skipped` attempt must NOT erase an earlier real failure. An
|
|
28
|
+
* auto-heal rerun in which the test skipped itself (e.g. a precondition
|
|
29
|
+
* guard fired because a prerequisite test wasn't part of the rerun) appends
|
|
30
|
+
* a skipped result after genuine failures — the failure stands.
|
|
31
|
+
* - A failed-then-passed history is `'flaky'` (Playwright's own term), not a
|
|
32
|
+
* plain pass — unless the self-heal signal is present, which wins.
|
|
33
|
+
* - A test whose final attempt matches its `expectedStatus` (e.g.
|
|
34
|
+
* `test.fail()` specs) is an `'expectedFailure'`: expected for stats and
|
|
35
|
+
* exit-code purposes, but never presented as a pass.
|
|
26
36
|
*/
|
|
27
37
|
export declare function statusOf(test: any): NormalizedStatus;
|
|
28
38
|
//# sourceMappingURL=reportWalk.d.ts.map
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* @fileoverview Shared helpers for walking a `DonobuReport` and classifying
|
|
4
|
-
* the tests inside it. Used by every renderer (HTML, Markdown, Slack)
|
|
5
|
-
* all agree on "what counts as self-healed"
|
|
4
|
+
* the tests inside it. Used by every renderer (HTML, Markdown, Slack) and the
|
|
5
|
+
* merge step's stats so they all agree on "what counts as self-healed",
|
|
6
|
+
* "what counts as flaky", and how suites nest.
|
|
6
7
|
*/
|
|
7
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
9
|
exports.collectSpecs = collectSpecs;
|
|
@@ -32,28 +33,77 @@ function isSelfHealed(test) {
|
|
|
32
33
|
const hasAnnotation = annotations.some((a) => a?.type === 'self-healed');
|
|
33
34
|
return hasAnnotation || test?.donobuStatus === 'healed';
|
|
34
35
|
}
|
|
36
|
+
const FAILURE_RESULT_STATUSES = new Set(['failed', 'timedOut', 'interrupted']);
|
|
35
37
|
/**
|
|
36
|
-
* Derive the display status for a test
|
|
37
|
-
*
|
|
38
|
-
*
|
|
38
|
+
* Derive the display status for a test from its full attempt history, not
|
|
39
|
+
* just the final attempt. The distinctions that matter:
|
|
40
|
+
*
|
|
41
|
+
* - A trailing `skipped` attempt must NOT erase an earlier real failure. An
|
|
42
|
+
* auto-heal rerun in which the test skipped itself (e.g. a precondition
|
|
43
|
+
* guard fired because a prerequisite test wasn't part of the rerun) appends
|
|
44
|
+
* a skipped result after genuine failures — the failure stands.
|
|
45
|
+
* - A failed-then-passed history is `'flaky'` (Playwright's own term), not a
|
|
46
|
+
* plain pass — unless the self-heal signal is present, which wins.
|
|
47
|
+
* - A test whose final attempt matches its `expectedStatus` (e.g.
|
|
48
|
+
* `test.fail()` specs) is an `'expectedFailure'`: expected for stats and
|
|
49
|
+
* exit-code purposes, but never presented as a pass.
|
|
39
50
|
*/
|
|
40
51
|
function statusOf(test) {
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
52
|
+
const results = test?.results ?? [];
|
|
53
|
+
const lastResult = results.at?.(-1);
|
|
54
|
+
// No attempts recorded — Playwright reports statically-skipped tests this
|
|
55
|
+
// way (and `test.status === 'skipped'` for runtime skips with no results).
|
|
56
|
+
if (!lastResult) {
|
|
44
57
|
return 'skipped';
|
|
45
58
|
}
|
|
46
59
|
if (isSelfHealed(test)) {
|
|
47
60
|
return 'healed';
|
|
48
61
|
}
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
const expectedStatus = test?.expectedStatus;
|
|
63
|
+
// An attempt "succeeded" when it passed outright, or when it landed on the
|
|
64
|
+
// declared expected outcome of a `test.fail()`-style spec. The two are kept
|
|
65
|
+
// distinct end-to-end: expected failures count as expected for stats and
|
|
66
|
+
// exit codes, but are never presented as passes.
|
|
67
|
+
const successKindOf = (result) => {
|
|
68
|
+
if (result?.status === 'passed') {
|
|
69
|
+
return 'passed';
|
|
70
|
+
}
|
|
71
|
+
if (expectedStatus !== undefined &&
|
|
72
|
+
expectedStatus !== 'passed' &&
|
|
73
|
+
expectedStatus !== 'skipped' &&
|
|
74
|
+
result?.status === expectedStatus) {
|
|
75
|
+
return 'expectedFailure';
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
};
|
|
79
|
+
const hadRealFailure = results.some((r) => FAILURE_RESULT_STATUSES.has(r?.status) && successKindOf(r) === null);
|
|
80
|
+
// A success anywhere in the history. Within a single run retries stop at
|
|
81
|
+
// the first success, so a success followed by a failure/skip only arises
|
|
82
|
+
// from a merged auto-heal rerun — and the rerun is advisory: it may flip a
|
|
83
|
+
// failure to healed but never an initial success to failed/skipped.
|
|
84
|
+
const priorSuccess = results.map(successKindOf).find((kind) => kind !== null) ?? null;
|
|
85
|
+
// Skips are handled first: a runtime-skipped test has
|
|
86
|
+
// `expectedStatus: 'skipped'` and must stay skipped.
|
|
87
|
+
if (lastResult.status === 'skipped') {
|
|
88
|
+
// A skip can't launder an earlier failure; surface the worst attempt.
|
|
89
|
+
if (hadRealFailure) {
|
|
90
|
+
const failure = results.find((r) => FAILURE_RESULT_STATUSES.has(r?.status) && successKindOf(r) === null);
|
|
91
|
+
return failure.status;
|
|
92
|
+
}
|
|
93
|
+
return priorSuccess ?? 'skipped';
|
|
94
|
+
}
|
|
95
|
+
const lastSuccess = successKindOf(lastResult);
|
|
96
|
+
if (lastSuccess === 'passed') {
|
|
97
|
+
return hadRealFailure ? 'flaky' : 'passed';
|
|
98
|
+
}
|
|
99
|
+
if (lastSuccess === 'expectedFailure') {
|
|
100
|
+
return 'expectedFailure';
|
|
101
|
+
}
|
|
102
|
+
switch (lastResult.status) {
|
|
52
103
|
case 'failed':
|
|
53
104
|
case 'timedOut':
|
|
54
|
-
case 'skipped':
|
|
55
105
|
case 'interrupted':
|
|
56
|
-
return status;
|
|
106
|
+
return priorSuccess ?? lastResult.status;
|
|
57
107
|
default:
|
|
58
108
|
return 'unknown';
|
|
59
109
|
}
|
|
@@ -20,6 +20,12 @@ export declare function extendPage(page: Page, options?: {
|
|
|
20
20
|
flowId?: string;
|
|
21
21
|
visualCueDurationMs?: number;
|
|
22
22
|
cacheFilepath?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Spec file this page is serving. Used to decide whether the file-scoped
|
|
25
|
+
* Page.AI cache invalidation (`DONOBU_PAGE_AI_CLEAR_CACHE_FILES`) applies
|
|
26
|
+
* to this context.
|
|
27
|
+
*/
|
|
28
|
+
specFilePath?: string;
|
|
23
29
|
envVars?: string[];
|
|
24
30
|
gptClient?: GptClient;
|
|
25
31
|
headless?: boolean;
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
exports.extendPage = extendPage;
|
|
4
7
|
const crypto_1 = require("crypto");
|
|
8
|
+
const path_1 = __importDefault(require("path"));
|
|
5
9
|
const v4_1 = require("zod/v4");
|
|
6
10
|
const GptClient_1 = require("../../clients/GptClient");
|
|
7
11
|
const VercelAiGptClient_1 = require("../../clients/VercelAiGptClient");
|
|
@@ -60,6 +64,25 @@ function resolveBaseUrl(page, url) {
|
|
|
60
64
|
// Donobu page extension helpers: decorate Playwright pages with Donobu behaviors and keep one
|
|
61
65
|
// coherent flow (and persistence record) per browser context so new tabs share state safely.
|
|
62
66
|
const PLACEHOLDER_FLOW_URL = 'https://example.com';
|
|
67
|
+
/**
|
|
68
|
+
* Whether Page.AI cache entries should be bypassed and invalidated for this
|
|
69
|
+
* context. Two knobs:
|
|
70
|
+
* - `DONOBU_PAGE_AI_CLEAR_CACHE` — run-wide, set by `--clear-ai-cache`.
|
|
71
|
+
* - `DONOBU_PAGE_AI_CLEAR_CACHE_FILES` — JSON array of spec paths, set by the
|
|
72
|
+
* auto-heal rerun so only heal-target spec files regenerate selectors while
|
|
73
|
+
* other re-running tests (serial prerequisites) keep their cache replay.
|
|
74
|
+
*/
|
|
75
|
+
function shouldClearPageAiCache(specFilePath) {
|
|
76
|
+
if (MiscUtils_1.MiscUtils.yn(envVars_1.env.data.DONOBU_PAGE_AI_CLEAR_CACHE)) {
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
const files = envVars_1.env.data.DONOBU_PAGE_AI_CLEAR_CACHE_FILES;
|
|
80
|
+
if (!files?.length || !specFilePath) {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
const resolved = path_1.default.resolve(specFilePath);
|
|
84
|
+
return files.some((file) => path_1.default.resolve(file) === resolved);
|
|
85
|
+
}
|
|
63
86
|
// Cache the shared Donobu state per browser context so every tab in that context reuses the same
|
|
64
87
|
// flow metadata, persistence, GPT client, and visualizer. WeakMap ensures cleanup when contexts die.
|
|
65
88
|
const contextSharedState = new WeakMap();
|
|
@@ -137,7 +160,7 @@ async function extendPage(page, options) {
|
|
|
137
160
|
gptClient: resolvedGptClient,
|
|
138
161
|
controlPanelFactory: options?.controlPanelFactory,
|
|
139
162
|
runtimeDirectives: {
|
|
140
|
-
clearPageAiCache:
|
|
163
|
+
clearPageAiCache: shouldClearPageAiCache(options?.specFilePath),
|
|
141
164
|
},
|
|
142
165
|
tbdSessions: [],
|
|
143
166
|
aiInvocations: [],
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Runtime gate for auto-heal reruns.
|
|
3
|
+
*
|
|
4
|
+
* The auto-heal rerun is launched with the same project-level arguments as the
|
|
5
|
+
* initial run so Playwright's scheduling (declared project `dependencies`,
|
|
6
|
+
* workers, ordering) behaves with full fidelity. Within the targeted projects,
|
|
7
|
+
* however, only the tests the heal actually needs should execute. This module
|
|
8
|
+
* enforces that from an auto fixture that runs before any browser fixture:
|
|
9
|
+
* when `DONOBU_AUTO_HEAL_PLAN_PATH` is set, every test not in the rerun plan
|
|
10
|
+
* skips immediately — before a context or page is created — annotated with
|
|
11
|
+
* `donobu-heal-skip-replay` so the merge step drops the entry and leaves the
|
|
12
|
+
* initial run's result untouched.
|
|
13
|
+
*
|
|
14
|
+
* What runs during a heal rerun (the "declared signals only" policy):
|
|
15
|
+
* - Heal targets (the failed tests with actionable treatment plans).
|
|
16
|
+
* - For a target inside a `test.describe.serial` scope (or a file marked
|
|
17
|
+
* serial via `test.describe.configure({ mode: 'serial' })`): the other
|
|
18
|
+
* serial-scoped tests in that file. Serial mode is Playwright's declared
|
|
19
|
+
* intra-file ordering contract — Playwright itself re-runs whole serial
|
|
20
|
+
* groups on retry, and we mirror that. The orchestrator expands the plan
|
|
21
|
+
* with these companions before the rerun (see
|
|
22
|
+
* `expandTargetsWithSerialCompanions`) using the `serialScoped` flags the
|
|
23
|
+
* Donobu reporter recorded during the initial run — the runner process
|
|
24
|
+
* sees the suite tree; the worker (where this gate runs) does not.
|
|
25
|
+
* - Declared dependency projects, which Playwright always runs in full.
|
|
26
|
+
*
|
|
27
|
+
* Implicit ordering (checkpoint files between plain tests, cross-file state
|
|
28
|
+
* with `workers: 1`) is deliberately NOT honored: tests relying on it will
|
|
29
|
+
* skip themselves during the rerun and surface as honest failures with
|
|
30
|
+
* guidance to declare the dependency.
|
|
31
|
+
*
|
|
32
|
+
* The gate runs at test runtime rather than collection time so Playwright's
|
|
33
|
+
* test-location attribution stays untouched (a collection-time wrapper would
|
|
34
|
+
* become every test's reported call site, breaking the merge's file-based
|
|
35
|
+
* matching) and so the plan can be matched against `testInfo.file`/`title`
|
|
36
|
+
* exactly instead of via stack inspection.
|
|
37
|
+
*/
|
|
38
|
+
import type { TestInfo } from '@playwright/test';
|
|
39
|
+
/** Shape of the plan file the auto-heal orchestrator writes before the rerun. */
|
|
40
|
+
export interface HealRerunPlan {
|
|
41
|
+
targets: Array<{
|
|
42
|
+
/** Spec file path; absolute, or relative to the rerun's CWD. */
|
|
43
|
+
file: string;
|
|
44
|
+
title: string;
|
|
45
|
+
projectName?: string;
|
|
46
|
+
}>;
|
|
47
|
+
}
|
|
48
|
+
/** Targets indexed by absolute spec path for O(1) per-test decisions. */
|
|
49
|
+
export type HealRerunPlanIndex = Map<string, Set<string>>;
|
|
50
|
+
export declare function buildPlanIndex(plan: HealRerunPlan): HealRerunPlanIndex;
|
|
51
|
+
/**
|
|
52
|
+
* Pure decision: should the test in `file` with `title` actually execute
|
|
53
|
+
* during the heal rerun? The plan is fully explicit — serial companions were
|
|
54
|
+
* already expanded into it by the orchestrator.
|
|
55
|
+
*/
|
|
56
|
+
export declare function shouldRunDuringHealRerun(params: {
|
|
57
|
+
index: HealRerunPlanIndex;
|
|
58
|
+
file: string;
|
|
59
|
+
title: string;
|
|
60
|
+
}): boolean;
|
|
61
|
+
/**
|
|
62
|
+
* Expand heal targets with their `describe.serial` siblings, using the
|
|
63
|
+
* `serialScoped` flags the Donobu reporter recorded in the initial run's
|
|
64
|
+
* report. Companions live in the same file as their target by construction
|
|
65
|
+
* (serial scopes are intra-file), so they inherit the target's absolute file
|
|
66
|
+
* path; report file paths are rootDir-relative, hence the suffix match.
|
|
67
|
+
*
|
|
68
|
+
* Degrades to the unexpanded targets when the report (or the flags) are
|
|
69
|
+
* unavailable — serial chains then surface as honest not-reattempted
|
|
70
|
+
* failures instead of healing.
|
|
71
|
+
*/
|
|
72
|
+
export declare function expandTargetsWithSerialCompanions(targets: HealRerunPlan['targets'], initialReport: {
|
|
73
|
+
suites?: unknown;
|
|
74
|
+
} | null): HealRerunPlan['targets'];
|
|
75
|
+
/** Test-only: reset the memoized plan so each test can load its own. */
|
|
76
|
+
export declare function resetHealRerunPlanCacheForTesting(): void;
|
|
77
|
+
/**
|
|
78
|
+
* Called from the Donobu auto fixture before any browser fixture initializes.
|
|
79
|
+
* Outside heal reruns this is a no-op. During a rerun, tests outside the plan
|
|
80
|
+
* are annotated and skipped on the spot — no context, no page, no cost.
|
|
81
|
+
*/
|
|
82
|
+
export declare function maybeSkipForHealRerun(testInfo: TestInfo, options?: {
|
|
83
|
+
planPath?: string;
|
|
84
|
+
}): void;
|
|
85
|
+
//# sourceMappingURL=healRerunGate.d.ts.map
|