codeceptjs 4.0.0-rc.2 → 4.0.0-rc.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -27
- package/bin/mcp-server.js +610 -0
- package/docs/webapi/appendField.mustache +5 -0
- package/docs/webapi/attachFile.mustache +12 -0
- package/docs/webapi/checkOption.mustache +1 -1
- package/docs/webapi/clearField.mustache +5 -0
- package/docs/webapi/dontSeeElement.mustache +4 -0
- package/docs/webapi/dontSeeInField.mustache +5 -0
- package/docs/webapi/fillField.mustache +5 -0
- package/docs/webapi/moveCursorTo.mustache +5 -1
- package/docs/webapi/seeElement.mustache +4 -0
- package/docs/webapi/seeInField.mustache +5 -0
- package/docs/webapi/selectOption.mustache +5 -0
- package/docs/webapi/uncheckOption.mustache +1 -1
- package/lib/codecept.js +20 -17
- package/lib/command/init.js +0 -3
- package/lib/command/run-workers.js +1 -0
- package/lib/container.js +19 -4
- package/lib/element/WebElement.js +52 -0
- package/lib/helper/Appium.js +8 -8
- package/lib/helper/Playwright.js +169 -87
- package/lib/helper/Puppeteer.js +181 -64
- package/lib/helper/WebDriver.js +141 -53
- package/lib/helper/errors/MultipleElementsFound.js +27 -110
- package/lib/helper/scripts/dropFile.js +11 -0
- package/lib/html.js +14 -1
- package/lib/listener/globalRetry.js +32 -6
- package/lib/mocha/cli.js +10 -0
- package/lib/plugin/aiTrace.js +464 -0
- package/lib/plugin/retryFailedStep.js +28 -19
- package/lib/plugin/stepByStepReport.js +5 -1
- package/lib/utils.js +48 -0
- package/lib/workers.js +49 -7
- package/package.json +5 -3
- package/lib/listener/enhancedGlobalRetry.js +0 -110
- package/lib/plugin/enhancedRetryFailedStep.js +0 -99
- package/lib/plugin/htmlReporter.js +0 -3648
- package/lib/retryCoordinator.js +0 -207
- package/typings/promiseBasedTypes.d.ts +0 -9469
- package/typings/types.d.ts +0 -11402
|
@@ -5,16 +5,27 @@ import { isNotSet } from '../utils.js'
|
|
|
5
5
|
|
|
6
6
|
const hooks = ['Before', 'After', 'BeforeSuite', 'AfterSuite']
|
|
7
7
|
|
|
8
|
+
const RETRY_PRIORITIES = {
|
|
9
|
+
MANUAL_STEP: 100,
|
|
10
|
+
STEP_PLUGIN: 50,
|
|
11
|
+
SCENARIO_CONFIG: 30,
|
|
12
|
+
FEATURE_CONFIG: 20,
|
|
13
|
+
HOOK_CONFIG: 10,
|
|
14
|
+
}
|
|
15
|
+
|
|
8
16
|
export default function () {
|
|
9
17
|
event.dispatcher.on(event.suite.before, suite => {
|
|
10
18
|
let retryConfig = Config.get('retry')
|
|
11
19
|
if (!retryConfig) return
|
|
12
20
|
|
|
13
21
|
if (Number.isInteger(+retryConfig)) {
|
|
14
|
-
// is number
|
|
15
22
|
const retryNum = +retryConfig
|
|
16
23
|
output.log(`Retries: ${retryNum}`)
|
|
17
|
-
|
|
24
|
+
|
|
25
|
+
if (suite.retries() === -1 || (suite.opts.retryPriority || 0) <= RETRY_PRIORITIES.FEATURE_CONFIG) {
|
|
26
|
+
suite.retries(retryNum)
|
|
27
|
+
suite.opts.retryPriority = RETRY_PRIORITIES.FEATURE_CONFIG
|
|
28
|
+
}
|
|
18
29
|
return
|
|
19
30
|
}
|
|
20
31
|
|
|
@@ -30,11 +41,18 @@ export default function () {
|
|
|
30
41
|
hooks
|
|
31
42
|
.filter(hook => !!config[hook])
|
|
32
43
|
.forEach(hook => {
|
|
33
|
-
|
|
44
|
+
const retryKey = `retry${hook}`
|
|
45
|
+
if (isNotSet(suite.opts[retryKey])) {
|
|
46
|
+
suite.opts[retryKey] = config[hook]
|
|
47
|
+
suite.opts[`${retryKey}Priority`] = RETRY_PRIORITIES.HOOK_CONFIG
|
|
48
|
+
}
|
|
34
49
|
})
|
|
35
50
|
|
|
36
51
|
if (config.Feature) {
|
|
37
|
-
if (
|
|
52
|
+
if (suite.retries() === -1 || (suite.opts.retryPriority || 0) <= RETRY_PRIORITIES.FEATURE_CONFIG) {
|
|
53
|
+
suite.retries(config.Feature)
|
|
54
|
+
suite.opts.retryPriority = RETRY_PRIORITIES.FEATURE_CONFIG
|
|
55
|
+
}
|
|
38
56
|
}
|
|
39
57
|
|
|
40
58
|
output.log(`Retries: ${JSON.stringify(config)}`)
|
|
@@ -46,7 +64,10 @@ export default function () {
|
|
|
46
64
|
if (!retryConfig) return
|
|
47
65
|
|
|
48
66
|
if (Number.isInteger(+retryConfig)) {
|
|
49
|
-
if (test.retries() === -1)
|
|
67
|
+
if (test.retries() === -1) {
|
|
68
|
+
test.retries(retryConfig)
|
|
69
|
+
test.opts.retryPriority = RETRY_PRIORITIES.SCENARIO_CONFIG
|
|
70
|
+
}
|
|
50
71
|
return
|
|
51
72
|
}
|
|
52
73
|
|
|
@@ -62,9 +83,14 @@ export default function () {
|
|
|
62
83
|
}
|
|
63
84
|
|
|
64
85
|
if (config.Scenario) {
|
|
65
|
-
if (test.retries() === -1
|
|
86
|
+
if (test.retries() === -1 || (test.opts.retryPriority || 0) <= RETRY_PRIORITIES.SCENARIO_CONFIG) {
|
|
87
|
+
test.retries(config.Scenario)
|
|
88
|
+
test.opts.retryPriority = RETRY_PRIORITIES.SCENARIO_CONFIG
|
|
89
|
+
}
|
|
66
90
|
output.log(`Retries: ${config.Scenario}`)
|
|
67
91
|
}
|
|
68
92
|
}
|
|
69
93
|
})
|
|
70
94
|
}
|
|
95
|
+
|
|
96
|
+
export { RETRY_PRIORITIES }
|
package/lib/mocha/cli.js
CHANGED
|
@@ -202,6 +202,16 @@ class Cli extends Base {
|
|
|
202
202
|
|
|
203
203
|
// failures
|
|
204
204
|
if (stats.failures) {
|
|
205
|
+
for (const test of this.failures) {
|
|
206
|
+
if (test.err && typeof test.err.fetchDetails === 'function') {
|
|
207
|
+
try {
|
|
208
|
+
await test.err.fetchDetails()
|
|
209
|
+
} catch (e) {
|
|
210
|
+
// ignore fetch errors
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
205
215
|
// append step traces
|
|
206
216
|
this.failures = this.failures.map(test => {
|
|
207
217
|
// we will change the stack trace, so we need to clone the test
|
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
import crypto from 'crypto'
|
|
2
|
+
import fs from 'fs'
|
|
3
|
+
import { mkdirp } from 'mkdirp'
|
|
4
|
+
import path from 'path'
|
|
5
|
+
import { fileURLToPath } from 'url'
|
|
6
|
+
|
|
7
|
+
import Container from '../container.js'
|
|
8
|
+
import recorder from '../recorder.js'
|
|
9
|
+
import event from '../event.js'
|
|
10
|
+
import output from '../output.js'
|
|
11
|
+
import { deleteDir, clearString } from '../utils.js'
|
|
12
|
+
import colors from 'chalk'
|
|
13
|
+
|
|
14
|
+
const supportedHelpers = Container.STANDARD_ACTING_HELPERS
|
|
15
|
+
|
|
16
|
+
const defaultConfig = {
|
|
17
|
+
deleteSuccessful: false,
|
|
18
|
+
fullPageScreenshots: false,
|
|
19
|
+
output: global.output_dir,
|
|
20
|
+
captureHTML: true,
|
|
21
|
+
captureARIA: true,
|
|
22
|
+
captureBrowserLogs: true,
|
|
23
|
+
captureHTTP: true,
|
|
24
|
+
captureDebugOutput: true,
|
|
25
|
+
ignoreSteps: [],
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
*
|
|
30
|
+
* Generates AI-friendly trace files for debugging with AI agents.
|
|
31
|
+
* This plugin creates a markdown file with test execution logs and links to all artifacts
|
|
32
|
+
* (screenshots, HTML, ARIA snapshots, browser logs, HTTP requests) for each step.
|
|
33
|
+
*
|
|
34
|
+
* #### Configuration
|
|
35
|
+
*
|
|
36
|
+
* ```js
|
|
37
|
+
* "plugins": {
|
|
38
|
+
* "aiTrace": {
|
|
39
|
+
* "enabled": true
|
|
40
|
+
* }
|
|
41
|
+
* }
|
|
42
|
+
* ```
|
|
43
|
+
*
|
|
44
|
+
* Possible config options:
|
|
45
|
+
*
|
|
46
|
+
* * `deleteSuccessful`: delete traces for successfully executed tests. Default: false.
|
|
47
|
+
* * `fullPageScreenshots`: should full page screenshots be used. Default: false.
|
|
48
|
+
* * `output`: a directory where traces should be stored. Default: `output`.
|
|
49
|
+
* * `captureHTML`: capture HTML for each step. Default: true.
|
|
50
|
+
* * `captureARIA`: capture ARIA snapshot for each step. Default: true.
|
|
51
|
+
* * `captureBrowserLogs`: capture browser console logs. Default: true.
|
|
52
|
+
* * `captureHTTP`: capture HTTP requests (requires `trace` or `recordHar` enabled in helper config). Default: true.
|
|
53
|
+
* * `captureDebugOutput`: capture CodeceptJS debug output. Default: true.
|
|
54
|
+
* * `ignoreSteps`: steps to ignore in trace. Array of RegExps is expected.
|
|
55
|
+
*
|
|
56
|
+
* @param {*} config
|
|
57
|
+
*/
|
|
58
|
+
export default function (config) {
|
|
59
|
+
const helpers = Container.helpers()
|
|
60
|
+
let helper
|
|
61
|
+
|
|
62
|
+
config = Object.assign(defaultConfig, config)
|
|
63
|
+
|
|
64
|
+
for (const helperName of supportedHelpers) {
|
|
65
|
+
if (Object.keys(helpers).indexOf(helperName) > -1) {
|
|
66
|
+
helper = helpers[helperName]
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!helper) {
|
|
71
|
+
output.warn('aiTrace plugin: No supported helper found (Playwright, Puppeteer, WebDriver). Plugin disabled.')
|
|
72
|
+
return
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
let dir
|
|
76
|
+
let stepNum
|
|
77
|
+
let steps = []
|
|
78
|
+
let debugOutput = []
|
|
79
|
+
let error
|
|
80
|
+
let savedSteps = new Set()
|
|
81
|
+
let currentTest = null
|
|
82
|
+
let testStartTime
|
|
83
|
+
let currentUrl = null
|
|
84
|
+
let testFailed = false
|
|
85
|
+
let firstFailedStepSaved = false
|
|
86
|
+
|
|
87
|
+
const reportDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output
|
|
88
|
+
|
|
89
|
+
if (config.captureDebugOutput) {
|
|
90
|
+
const originalDebug = output.debug
|
|
91
|
+
output.debug = function (...args) {
|
|
92
|
+
debugOutput.push(args.join(' '))
|
|
93
|
+
originalDebug.apply(output, args)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
event.dispatcher.on(event.suite.before, suite => {
|
|
98
|
+
stepNum = -1
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
event.dispatcher.on(event.test.before, test => {
|
|
102
|
+
let title
|
|
103
|
+
try {
|
|
104
|
+
title = test.fullTitle ? test.fullTitle() : test.title
|
|
105
|
+
} catch (err) {
|
|
106
|
+
title = test.title
|
|
107
|
+
}
|
|
108
|
+
const testTitle = clearString(title).slice(0, 200)
|
|
109
|
+
const uniqueHash = crypto
|
|
110
|
+
.createHash('sha256')
|
|
111
|
+
.update(test.file + test.title)
|
|
112
|
+
.digest('hex')
|
|
113
|
+
.slice(0, 8)
|
|
114
|
+
dir = path.join(reportDir, `trace_${testTitle}_${uniqueHash}`)
|
|
115
|
+
mkdirp.sync(dir)
|
|
116
|
+
deleteDir(dir)
|
|
117
|
+
mkdirp.sync(dir)
|
|
118
|
+
stepNum = 0
|
|
119
|
+
error = null
|
|
120
|
+
steps = []
|
|
121
|
+
debugOutput = []
|
|
122
|
+
savedSteps.clear()
|
|
123
|
+
currentTest = test
|
|
124
|
+
testStartTime = Date.now()
|
|
125
|
+
currentUrl = null
|
|
126
|
+
testFailed = false
|
|
127
|
+
firstFailedStepSaved = false
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
event.dispatcher.on(event.step.after, step => {
|
|
131
|
+
if (!currentTest) return
|
|
132
|
+
if (step.status === 'failed') {
|
|
133
|
+
testFailed = true
|
|
134
|
+
}
|
|
135
|
+
if (step.status === 'queued' && testFailed) {
|
|
136
|
+
output.debug(`aiTrace: Skipping queued step "${step.toString()}" - testFailed: ${testFailed}`)
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
if (step.status === 'failed' && firstFailedStepSaved) {
|
|
140
|
+
output.debug(`aiTrace: Skipping failed step "${step.toString()}" - already handled by step.failed event`)
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
const stepPersistPromise = persistStep(step).catch(err => {
|
|
144
|
+
output.debug(`aiTrace: Error saving step: ${err.message}`)
|
|
145
|
+
})
|
|
146
|
+
recorder.add(`wait aiTrace step persistence: ${step.toString()}`, () => stepPersistPromise, true)
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
event.dispatcher.on(event.step.failed, async step => {
|
|
150
|
+
if (!currentTest) return
|
|
151
|
+
if (step.status === 'queued' && testFailed) {
|
|
152
|
+
output.debug(`aiTrace: Skipping queued failed step "${step.toString()}" - testFailed: ${testFailed}`)
|
|
153
|
+
return
|
|
154
|
+
}
|
|
155
|
+
if (firstFailedStepSaved) {
|
|
156
|
+
output.debug(`aiTrace: Skipping subsequent failed step "${step.toString()}" - already saved first failed step`)
|
|
157
|
+
return
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const stepKey = step.toString()
|
|
161
|
+
if (savedSteps.has(stepKey)) {
|
|
162
|
+
const existingStep = steps.find(s => s.step === stepKey)
|
|
163
|
+
if (!existingStep) {
|
|
164
|
+
output.debug(`aiTrace: Step "${stepKey}" marked as saved but not found in steps array`)
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
existingStep.status = 'failed'
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
await captureArtifactsForStep(step, existingStep, existingStep.prefix)
|
|
171
|
+
} catch (err) {
|
|
172
|
+
output.debug(`aiTrace: Error updating failed step: ${err.message}`)
|
|
173
|
+
}
|
|
174
|
+
} else {
|
|
175
|
+
if (stepNum === -1) return
|
|
176
|
+
if (isStepIgnored(step)) return
|
|
177
|
+
if (step.metaStep && step.metaStep.name === 'BeforeSuite') return
|
|
178
|
+
|
|
179
|
+
const stepPrefix = generateStepPrefix(step, stepNum)
|
|
180
|
+
stepNum++
|
|
181
|
+
|
|
182
|
+
const stepData = {
|
|
183
|
+
step: stepKey,
|
|
184
|
+
status: 'failed',
|
|
185
|
+
prefix: stepPrefix,
|
|
186
|
+
artifacts: {},
|
|
187
|
+
meta: {},
|
|
188
|
+
debugOutput: [],
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (step.startTime && step.endTime) {
|
|
192
|
+
stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
savedSteps.add(stepKey)
|
|
196
|
+
steps.push(stepData)
|
|
197
|
+
firstFailedStepSaved = true
|
|
198
|
+
|
|
199
|
+
try {
|
|
200
|
+
await captureArtifactsForStep(step, stepData, stepPrefix)
|
|
201
|
+
} catch (err) {
|
|
202
|
+
output.debug(`aiTrace: Error capturing failed step artifacts: ${err.message}`)
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
})
|
|
206
|
+
|
|
207
|
+
event.dispatcher.on(event.test.passed, test => {
|
|
208
|
+
if (config.deleteSuccessful) {
|
|
209
|
+
deleteDir(dir)
|
|
210
|
+
return
|
|
211
|
+
}
|
|
212
|
+
persist(test, 'passed')
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
|
|
216
|
+
if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') {
|
|
217
|
+
return
|
|
218
|
+
}
|
|
219
|
+
persist(test, 'failed')
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
async function persistStep(step) {
|
|
223
|
+
if (stepNum === -1) return
|
|
224
|
+
if (isStepIgnored(step)) return
|
|
225
|
+
if (step.metaStep && step.metaStep.name === 'BeforeSuite') return
|
|
226
|
+
|
|
227
|
+
const stepKey = step.toString()
|
|
228
|
+
|
|
229
|
+
if (savedSteps.has(stepKey)) {
|
|
230
|
+
const existingStep = steps.find(s => s.step === stepKey)
|
|
231
|
+
if (existingStep && step.status === 'failed') {
|
|
232
|
+
existingStep.status = 'failed'
|
|
233
|
+
step.artifacts = {}
|
|
234
|
+
await captureArtifactsForStep(step, existingStep, existingStep.prefix)
|
|
235
|
+
}
|
|
236
|
+
return
|
|
237
|
+
}
|
|
238
|
+
savedSteps.add(stepKey)
|
|
239
|
+
|
|
240
|
+
const stepPrefix = generateStepPrefix(step, stepNum)
|
|
241
|
+
stepNum++
|
|
242
|
+
|
|
243
|
+
const stepData = {
|
|
244
|
+
step: step.toString(),
|
|
245
|
+
status: step.status,
|
|
246
|
+
prefix: stepPrefix,
|
|
247
|
+
artifacts: {},
|
|
248
|
+
meta: {},
|
|
249
|
+
debugOutput: [],
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (step.startTime && step.endTime) {
|
|
253
|
+
stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (config.captureDebugOutput && debugOutput.length > 0) {
|
|
257
|
+
stepData.debugOutput = [...debugOutput]
|
|
258
|
+
debugOutput = []
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
await captureArtifactsForStep(step, stepData, stepPrefix)
|
|
262
|
+
steps.push(stepData)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
async function captureArtifactsForStep(step, stepData, stepPrefix) {
|
|
266
|
+
if (!step.artifacts) {
|
|
267
|
+
step.artifacts = {}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
let browserAvailable = true
|
|
271
|
+
|
|
272
|
+
try {
|
|
273
|
+
try {
|
|
274
|
+
if (helper.grabCurrentUrl) {
|
|
275
|
+
const url = await helper.grabCurrentUrl()
|
|
276
|
+
stepData.meta.url = url
|
|
277
|
+
currentUrl = url
|
|
278
|
+
}
|
|
279
|
+
} catch (err) {
|
|
280
|
+
browserAvailable = false
|
|
281
|
+
output.debug(`aiTrace: Browser unavailable, partial artifact capture: ${err.message}`)
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (step.artifacts?.screenshot) {
|
|
285
|
+
const screenshotPath = path.isAbsolute(step.artifacts.screenshot)
|
|
286
|
+
? step.artifacts.screenshot
|
|
287
|
+
: path.resolve(dir, step.artifacts.screenshot)
|
|
288
|
+
const screenshotFile = path.basename(screenshotPath)
|
|
289
|
+
stepData.artifacts.screenshot = screenshotFile
|
|
290
|
+
step.artifacts.screenshot = screenshotPath
|
|
291
|
+
|
|
292
|
+
if (!fs.existsSync(screenshotPath)) {
|
|
293
|
+
try {
|
|
294
|
+
await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots)
|
|
295
|
+
} catch (err) {
|
|
296
|
+
output.debug(`aiTrace: Could not save screenshot: ${err.message}`)
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
} else {
|
|
300
|
+
try {
|
|
301
|
+
const screenshotFile = `${stepPrefix}_screenshot.png`
|
|
302
|
+
const screenshotPath = path.join(dir, screenshotFile)
|
|
303
|
+
await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots)
|
|
304
|
+
|
|
305
|
+
stepData.artifacts.screenshot = screenshotFile
|
|
306
|
+
step.artifacts.screenshot = screenshotPath
|
|
307
|
+
} catch (err) {
|
|
308
|
+
output.debug(`aiTrace: Could not save screenshot: ${err.message}`)
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Save HTML
|
|
313
|
+
if (config.captureHTML && helper.grabSource && browserAvailable) {
|
|
314
|
+
if (!step.artifacts?.html) {
|
|
315
|
+
try {
|
|
316
|
+
const html = await helper.grabSource()
|
|
317
|
+
const htmlFile = `${stepPrefix}_page.html`
|
|
318
|
+
fs.writeFileSync(path.join(dir, htmlFile), html)
|
|
319
|
+
stepData.artifacts.html = htmlFile
|
|
320
|
+
} catch (err) {
|
|
321
|
+
output.debug(`aiTrace: Could not capture HTML: ${err.message}`)
|
|
322
|
+
}
|
|
323
|
+
} else {
|
|
324
|
+
stepData.artifacts.html = step.artifacts.html
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Save ARIA snapshot
|
|
329
|
+
if (config.captureARIA && helper.grabAriaSnapshot && browserAvailable) {
|
|
330
|
+
try {
|
|
331
|
+
const aria = await helper.grabAriaSnapshot()
|
|
332
|
+
const ariaFile = `${stepPrefix}_aria.txt`
|
|
333
|
+
fs.writeFileSync(path.join(dir, ariaFile), aria)
|
|
334
|
+
stepData.artifacts.aria = ariaFile
|
|
335
|
+
} catch (err) {
|
|
336
|
+
output.debug(`aiTrace: Could not capture ARIA snapshot: ${err.message}`)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Save browser logs
|
|
341
|
+
if (config.captureBrowserLogs && helper.grabBrowserLogs && browserAvailable) {
|
|
342
|
+
try {
|
|
343
|
+
const logs = await helper.grabBrowserLogs()
|
|
344
|
+
const logsFile = `${stepPrefix}_console.json`
|
|
345
|
+
fs.writeFileSync(path.join(dir, logsFile), JSON.stringify(logs || [], null, 2))
|
|
346
|
+
stepData.artifacts.console = logsFile
|
|
347
|
+
stepData.meta.consoleCount = logs ? logs.length : 0
|
|
348
|
+
} catch (err) {
|
|
349
|
+
output.debug(`aiTrace: Could not capture browser logs: ${err.message}`)
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
} catch (err) {
|
|
353
|
+
output.plugin(`aiTrace: Can't save step artifacts: ${err}`)
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function persist(test, status) {
|
|
358
|
+
if (!steps.length) {
|
|
359
|
+
output.debug('aiTrace: No steps to save in trace')
|
|
360
|
+
return
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const testDuration = ((Date.now() - testStartTime) / 1000).toFixed(2)
|
|
364
|
+
|
|
365
|
+
let markdown = `file: ${test.file || 'unknown'}\n`
|
|
366
|
+
markdown += `name: ${test.title}\n`
|
|
367
|
+
markdown += `time: ${testDuration}s\n`
|
|
368
|
+
markdown += `---\n\n`
|
|
369
|
+
|
|
370
|
+
if (status === 'failed') {
|
|
371
|
+
if (test.art && test.art.message) {
|
|
372
|
+
markdown += `Error: ${test.art.message}\n\n`
|
|
373
|
+
}
|
|
374
|
+
if (test.art && test.art.stack) {
|
|
375
|
+
markdown += `${test.art.stack}\n\n`
|
|
376
|
+
}
|
|
377
|
+
markdown += `---\n\n`
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (config.captureDebugOutput && debugOutput.length > 0) {
|
|
381
|
+
markdown += `CodeceptJS Debug Output:\n\n`
|
|
382
|
+
debugOutput.forEach(line => {
|
|
383
|
+
markdown += `> ${line}\n`
|
|
384
|
+
})
|
|
385
|
+
markdown += `\n---\n\n`
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
steps.forEach((stepData, index) => {
|
|
389
|
+
const stepAnchor = clearString(stepData.step).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50)
|
|
390
|
+
markdown += `### Step ${index + 1}: ${stepData.step}\n`
|
|
391
|
+
markdown += `<a id="${stepAnchor}"></a>\n`
|
|
392
|
+
|
|
393
|
+
if (stepData.meta.duration) {
|
|
394
|
+
markdown += ` > duration: ${stepData.meta.duration}\n`
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
if (stepData.meta.url) {
|
|
398
|
+
markdown += ` > navigated to ${stepData.meta.url}\n`
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (config.captureDebugOutput && stepData.debugOutput && stepData.debugOutput.length > 0) {
|
|
402
|
+
stepData.debugOutput.forEach(line => {
|
|
403
|
+
markdown += ` > ${line}\n`
|
|
404
|
+
})
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
if (stepData.artifacts.html) {
|
|
408
|
+
markdown += ` > [HTML](./${stepData.artifacts.html})\n`
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
if (stepData.artifacts.aria) {
|
|
412
|
+
markdown += ` > [ARIA Snapshot](./${stepData.artifacts.aria})\n`
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
if (stepData.artifacts.screenshot) {
|
|
416
|
+
markdown += ` > [Screenshot](./${stepData.artifacts.screenshot})\n`
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
if (stepData.artifacts.console) {
|
|
420
|
+
const count = stepData.meta.consoleCount || 0
|
|
421
|
+
markdown += ` > [Browser Logs](./${stepData.artifacts.console}) (${count} entries)\n`
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
if (config.captureHTTP) {
|
|
425
|
+
if (test.artifacts && test.artifacts.har) {
|
|
426
|
+
const harPath = path.relative(reportDir, test.artifacts.har)
|
|
427
|
+
markdown += ` > HTTP: see [HAR file](../${harPath}) for network requests\n`
|
|
428
|
+
} else if (test.artifacts && test.artifacts.trace) {
|
|
429
|
+
const tracePath = path.relative(reportDir, test.artifacts.trace)
|
|
430
|
+
markdown += ` > HTTP: see [Playwright trace](../${tracePath}) for network requests\n`
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
markdown += `\n`
|
|
435
|
+
})
|
|
436
|
+
|
|
437
|
+
const traceFile = path.join(dir, 'trace.md')
|
|
438
|
+
fs.writeFileSync(traceFile, markdown)
|
|
439
|
+
|
|
440
|
+
output.print(`🤖 AI Trace: ${colors.white.bold(`file://${traceFile}`)}`)
|
|
441
|
+
|
|
442
|
+
if (!test.artifacts) test.artifacts = {}
|
|
443
|
+
test.artifacts.aiTrace = traceFile
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function isStepIgnored(step) {
|
|
447
|
+
if (!config.ignoreSteps) return false
|
|
448
|
+
for (const pattern of config.ignoreSteps || []) {
|
|
449
|
+
if (step.name.match(pattern)) return true
|
|
450
|
+
}
|
|
451
|
+
return false
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function generateStepPrefix(step, index) {
|
|
455
|
+
const stepName = step.toString()
|
|
456
|
+
const cleanedName = clearString(stepName)
|
|
457
|
+
.replace(/[^a-zA-Z0-9_-]/g, '_')
|
|
458
|
+
.replace(/_{2,}/g, '_')
|
|
459
|
+
.slice(0, 80)
|
|
460
|
+
.trim()
|
|
461
|
+
|
|
462
|
+
return `${String(index).padStart(4, '0')}_${cleanedName}`
|
|
463
|
+
}
|
|
464
|
+
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import event from '../event.js'
|
|
2
|
-
|
|
3
2
|
import recorder from '../recorder.js'
|
|
4
|
-
|
|
5
3
|
import store from '../store.js'
|
|
6
4
|
|
|
7
5
|
const defaultConfig = {
|
|
@@ -9,6 +7,15 @@ const defaultConfig = {
|
|
|
9
7
|
defaultIgnoredSteps: ['amOnPage', 'wait*', 'send*', 'execute*', 'run*', 'have*'],
|
|
10
8
|
factor: 1.5,
|
|
11
9
|
ignoredSteps: [],
|
|
10
|
+
deferToScenarioRetries: true,
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const RETRY_PRIORITIES = {
|
|
14
|
+
MANUAL_STEP: 100,
|
|
15
|
+
STEP_PLUGIN: 50,
|
|
16
|
+
SCENARIO_CONFIG: 30,
|
|
17
|
+
FEATURE_CONFIG: 20,
|
|
18
|
+
HOOK_CONFIG: 10,
|
|
12
19
|
}
|
|
13
20
|
|
|
14
21
|
/**
|
|
@@ -49,6 +56,7 @@ const defaultConfig = {
|
|
|
49
56
|
* * `ignoredSteps` - an array for custom steps to ignore on retry. Use it to append custom steps to ignored list.
|
|
50
57
|
* You can use step names or step prefixes ending with `*`. As such, `wait*` will match all steps starting with `wait`.
|
|
51
58
|
* To append your own steps to ignore list - copy and paste a default steps list. Regexp values are accepted as well.
|
|
59
|
+
* * `deferToScenarioRetries` - when enabled (default), step retries are automatically disabled if scenario retries are configured to avoid excessive total retries.
|
|
52
60
|
*
|
|
53
61
|
* #### Example
|
|
54
62
|
*
|
|
@@ -88,73 +96,74 @@ export default function (config) {
|
|
|
88
96
|
if (!enableRetry) return
|
|
89
97
|
if (store.debugMode) return false
|
|
90
98
|
if (!store.autoRetries) return false
|
|
91
|
-
// Don't retry terminal errors (e.g., frame detachment errors)
|
|
92
99
|
if (err && err.isTerminal) return false
|
|
93
|
-
// Don't retry navigation errors that are known to be terminal
|
|
94
100
|
if (err && err.message && (err.message.includes('ERR_ABORTED') || err.message.includes('frame was detached') || err.message.includes('Target page, context or browser has been closed'))) return false
|
|
95
101
|
if (customWhen) return customWhen(err)
|
|
96
102
|
return true
|
|
97
103
|
}
|
|
98
104
|
config.when = when
|
|
99
105
|
|
|
100
|
-
// Ensure retry options are available before any steps run
|
|
101
106
|
if (!recorder.retries.find(r => r === config)) {
|
|
102
107
|
recorder.retries.push(config)
|
|
103
108
|
}
|
|
104
109
|
|
|
105
110
|
event.dispatcher.on(event.step.started, step => {
|
|
106
|
-
// if a step is ignored - return
|
|
107
111
|
for (const ignored of config.ignoredSteps) {
|
|
108
112
|
if (step.name === ignored) return
|
|
109
113
|
if (ignored instanceof RegExp) {
|
|
110
114
|
if (step.name.match(ignored)) return
|
|
111
115
|
} else if (ignored.indexOf('*') && step.name.startsWith(ignored.slice(0, -1))) return
|
|
112
116
|
}
|
|
113
|
-
enableRetry = true
|
|
117
|
+
enableRetry = true
|
|
114
118
|
})
|
|
115
119
|
|
|
116
|
-
// Disable retry only after a successful step; keep it enabled for failure so retry logic can act
|
|
117
120
|
event.dispatcher.on(event.step.passed, () => {
|
|
118
121
|
enableRetry = false
|
|
119
122
|
})
|
|
120
123
|
|
|
121
124
|
event.dispatcher.on(event.test.before, test => {
|
|
122
|
-
// pass disableRetryFailedStep is a preferred way to disable retries
|
|
123
|
-
// test.disableRetryFailedStep is used for backward compatibility
|
|
124
125
|
if (!test.opts) test.opts = {}
|
|
125
126
|
if (test.opts.disableRetryFailedStep || test.disableRetryFailedStep) {
|
|
126
127
|
store.autoRetries = false
|
|
127
|
-
return
|
|
128
|
+
return
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const scenarioRetries = typeof test.retries === 'function' ? test.retries() : -1
|
|
132
|
+
const stepRetryPriority = RETRY_PRIORITIES.STEP_PLUGIN
|
|
133
|
+
const scenarioPriority = test.opts.retryPriority || 0
|
|
134
|
+
|
|
135
|
+
if (scenarioRetries > 0 && config.deferToScenarioRetries !== false) {
|
|
136
|
+
store.autoRetries = false
|
|
137
|
+
return
|
|
128
138
|
}
|
|
129
139
|
|
|
130
|
-
// Don't apply plugin retry logic if there are already manual retries configured
|
|
131
|
-
// Check if any retry configs exist that aren't from this plugin
|
|
132
140
|
const hasManualRetries = recorder.retries.some(retry => retry !== config)
|
|
133
141
|
if (hasManualRetries) {
|
|
134
142
|
store.autoRetries = false
|
|
135
143
|
return
|
|
136
144
|
}
|
|
137
145
|
|
|
138
|
-
// this option is used to set the retries inside _before() block of helpers
|
|
139
146
|
store.autoRetries = true
|
|
140
147
|
test.opts.conditionalRetries = config.retries
|
|
141
|
-
|
|
148
|
+
test.opts.stepRetryPriority = stepRetryPriority
|
|
149
|
+
|
|
142
150
|
if (process.env.DEBUG_RETRY_PLUGIN) {
|
|
143
|
-
// eslint-disable-next-line no-console
|
|
144
151
|
console.log('[retryFailedStep] applying retries =', config.retries, 'for test', test.title)
|
|
145
152
|
}
|
|
146
153
|
recorder.retry(config)
|
|
147
154
|
})
|
|
148
155
|
|
|
149
|
-
// Fallback for environments where event.test.before wasn't emitted (runner scenarios)
|
|
150
156
|
event.dispatcher.on(event.test.started, test => {
|
|
151
157
|
if (test.opts?.disableRetryFailedStep || test.disableRetryFailedStep) return
|
|
152
158
|
|
|
153
|
-
// Don't apply plugin retry logic if there are already manual retries configured
|
|
154
|
-
// Check if any retry configs exist that aren't from this plugin
|
|
155
159
|
const hasManualRetries = recorder.retries.some(retry => retry !== config)
|
|
156
160
|
if (hasManualRetries) return
|
|
157
161
|
|
|
162
|
+
const scenarioRetries = typeof test.retries === 'function' ? test.retries() : -1
|
|
163
|
+
if (scenarioRetries > 0 && config.deferToScenarioRetries !== false) {
|
|
164
|
+
return
|
|
165
|
+
}
|
|
166
|
+
|
|
158
167
|
if (!store.autoRetries) {
|
|
159
168
|
store.autoRetries = true
|
|
160
169
|
test.opts.conditionalRetries = test.opts.conditionalRetries || config.retries
|
|
@@ -207,7 +207,11 @@ export default function (config) {
|
|
|
207
207
|
stepNum++
|
|
208
208
|
slides[fileName] = step
|
|
209
209
|
try {
|
|
210
|
-
|
|
210
|
+
const screenshotPath = path.join(dir, fileName)
|
|
211
|
+
await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots)
|
|
212
|
+
|
|
213
|
+
step.artifacts = step.artifacts || {}
|
|
214
|
+
step.artifacts.screenshot = screenshotPath
|
|
211
215
|
} catch (err) {
|
|
212
216
|
output.plugin(`Can't save step screenshot: ${err}`)
|
|
213
217
|
error = err
|