codeceptjs 4.0.0-rc.20 → 4.0.0-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mcp-server.js +7 -5
- package/docs/agents.md +32 -10
- package/docs/environment-variables.md +131 -0
- package/lib/codecept.js +1 -1
- package/lib/command/run-workers.js +0 -14
- package/lib/command/run.js +2 -16
- package/lib/command/utils.js +14 -0
- package/lib/heal.js +2 -0
- package/lib/helper/Playwright.js +5 -0
- package/lib/plugin/aiTrace.js +16 -13
- package/lib/plugin/analyze.js +5 -4
- package/lib/plugin/heal.js +3 -2
- package/lib/plugin/pageInfo.js +5 -7
- package/lib/plugin/retryFailedStep.js +4 -3
- package/lib/plugin/screencast.js +6 -4
- package/lib/workers.js +11 -3
- package/package.json +1 -1
package/bin/mcp-server.js
CHANGED
|
@@ -60,9 +60,7 @@ function aiTraceHint() {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
function applyMochaGrep(grep) {
|
|
63
|
-
if (
|
|
64
|
-
const mocha = typeof container.mocha === 'function' ? container.mocha() : container.mocha
|
|
65
|
-
if (mocha && typeof mocha.grep === 'function') mocha.grep(grep)
|
|
63
|
+
if (grep) container.mocha().grep(grep)
|
|
66
64
|
}
|
|
67
65
|
|
|
68
66
|
function pauseAtMatcher(pauseAt) {
|
|
@@ -401,14 +399,16 @@ async function cancelRun() {
|
|
|
401
399
|
abortRun = true
|
|
402
400
|
if (typeof pendingRunCleanup === 'function') { try { pendingRunCleanup() } catch {} }
|
|
403
401
|
if (pausedController) { try { pausedController.resolveContinue() } catch {} ; pausedController = null }
|
|
402
|
+
|
|
403
|
+
try { container.mocha().runner?.abort() } catch {}
|
|
404
|
+
|
|
404
405
|
if (pendingRunPromise) {
|
|
405
|
-
try { await
|
|
406
|
+
try { await pendingRunPromise.catch(() => {}) } catch {}
|
|
406
407
|
}
|
|
407
408
|
pendingRunPromise = null
|
|
408
409
|
pendingRunResults = null
|
|
409
410
|
pendingTestFile = null
|
|
410
411
|
pendingStepInfo = null
|
|
411
|
-
abortRun = false
|
|
412
412
|
return true
|
|
413
413
|
}
|
|
414
414
|
|
|
@@ -1032,6 +1032,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1032
1032
|
pendingRunCleanup = null
|
|
1033
1033
|
}
|
|
1034
1034
|
|
|
1035
|
+
abortRun = false
|
|
1035
1036
|
let runError = null
|
|
1036
1037
|
const runPromise = (async () => {
|
|
1037
1038
|
try {
|
|
@@ -1126,6 +1127,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1126
1127
|
pendingRunCleanup = null
|
|
1127
1128
|
}
|
|
1128
1129
|
|
|
1130
|
+
abortRun = false
|
|
1129
1131
|
let runError = null
|
|
1130
1132
|
const runPromise = (async () => {
|
|
1131
1133
|
try {
|
package/docs/agents.md
CHANGED
|
@@ -5,6 +5,38 @@ title: Agentic Testing
|
|
|
5
5
|
|
|
6
6
|
# Agentic Testing
|
|
7
7
|
|
|
8
|
+
## What Makes CodeceptJS Agent-friendly
|
|
9
|
+
|
|
10
|
+
CodeceptJS 4 is designed for agent testing. It ships with its own MCP and official skills that teach agents best practices in test automation—how to write tests, choose locators, test them live in the browser, and refactor to page objects.
|
|
11
|
+
|
|
12
|
+
Agents get full control over test and browser execution:
|
|
13
|
+
|
|
14
|
+
- **Full HTML access and ARIA snapshots.** Agents see buttons with icons, empty labels, and other elements Playwright MCP's accessibility tree omits.
|
|
15
|
+
|
|
16
|
+
- **Query HTML directly with CSS, ARIA, Semantic Locators or XPath** Agents can freely run XPath or CSS to efficiently search over HTML.
|
|
17
|
+
|
|
18
|
+
- **Browser logs and page state as files.** Agents read via native filesystem tools instead of extra requests, eliminating redundant context dumps.
|
|
19
|
+
|
|
20
|
+
- **Same locators for tests and agents.** Tests and agent scripts share identical selectors. No elements by reference indexes, no clicks by coordinates. Agents inherit battle-tested patterns without guessing.
|
|
21
|
+
|
|
22
|
+
- **Testing-first framework, not browser control.** CodeceptJS is built for testing, so agents stay focused on test scenarios instead of raw browser commands.
|
|
23
|
+
|
|
24
|
+
CodeceptJS is token-efficient: it stores HTML, ARIA, logs, and HTTP request data as files instead of streaming them through MCP. Agents read these files with their native shell tools—no extra API calls, no redundant context.
|
|
25
|
+
|
|
26
|
+
## The loop
|
|
27
|
+
|
|
28
|
+
Whether the agent is writing a new test or fixing an old one, it follows the same cycle.
|
|
29
|
+
|
|
30
|
+
1. **Open the page.** Run a stub test (new work) or set a breakpoint at the failing step (fix). The browser lands at the right starting point and yields control to the agent.
|
|
31
|
+
2. **Read the page.** MCP saves HTML, ARIA, and screenshot of the page to files (and the agent can call the `snapshot` tool to refresh them). The agent reads those files before deciding what to try next, controlling its token usage.
|
|
32
|
+
3. **Run a CodeceptJS command.** The agent tries `I.*` commands like `I.click('Add to cart')`, `I.fillField('Email', secret(process.env.EMAIL))`, `I.see('Confirmed')`. On success, that line goes into the test — same syntax.
|
|
33
|
+
4. **Check the result.** The response after each command shows the new page state. If the URL changed and the modal opened, the line goes into the verified sequence. If not, the agent reads the page again and tries a different locator or a wait.
|
|
34
|
+
5. **Move forward.** The agent looks at the new state and chooses the next command. Steps 2–4 repeat until the scenario is whole.
|
|
35
|
+
6. **Commit to the file.** The agent edits the test — replaces `pause()` (new tests) or the broken line (fixes) with the verified sequence — then reruns end-to-end and reads the trace to confirm.
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
## How It Works
|
|
39
|
+
|
|
8
40
|
CodeceptJS ships an **MCP server and a skillset** that lets an AI agent (Claude Code, Cursor, Codex, others) write and fix tests by driving the real browser. The agent runs the same `I.*` commands the test does, reads how the page responds, and only commits the lines that succeeded.
|
|
9
41
|
|
|
10
42
|
## Why MCP
|
|
@@ -30,16 +62,6 @@ This lets the agent get a test working in one iteration. The agent can live-writ
|
|
|
30
62
|
|
|
31
63
|
The MCP server is the agent-facing equivalent of the `pause()` REPL — same access, driven by tool calls instead of keystrokes. Full tool reference at [/mcp](/mcp).
|
|
32
64
|
|
|
33
|
-
## The loop
|
|
34
|
-
|
|
35
|
-
Whether the agent is writing a new test or fixing an old one, it follows the same cycle.
|
|
36
|
-
|
|
37
|
-
1. **Open the page.** Run a stub test (new work) or set a breakpoint at the failing step (fix). The browser lands at the right starting point and yields control to the agent.
|
|
38
|
-
2. **Read the page.** MCP saves HTML, ARIA, and screenshot of the page to files (and the agent can call the `snapshot` tool to refresh them). The agent reads those files before deciding what to try next, controlling its token usage.
|
|
39
|
-
3. **Run a CodeceptJS command.** The agent tries `I.*` commands like `I.click('Add to cart')`, `I.fillField('Email', secret(process.env.EMAIL))`, `I.see('Confirmed')`. On success, that line goes into the test — same syntax.
|
|
40
|
-
4. **Check the result.** The response after each command shows the new page state. If the URL changed and the modal opened, the line goes into the verified sequence. If not, the agent reads the page again and tries a different locator or a wait.
|
|
41
|
-
5. **Move forward.** The agent looks at the new state and chooses the next command. Steps 2–4 repeat until the scenario is whole.
|
|
42
|
-
6. **Commit to the file.** The agent edits the test — replaces `pause()` (new tests) or the broken line (fixes) with the verified sequence — then reruns end-to-end and reads the trace to confirm.
|
|
43
65
|
|
|
44
66
|
## How the agent reads the page
|
|
45
67
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
permalink: /environment-variables
|
|
3
|
+
title: Environment Variables
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Environment Variables
|
|
7
|
+
|
|
8
|
+
CodeceptJS reads several environment variables that change runtime behavior. They are useful for tuning CI runs, enabling extra logging, configuring the MCP server, or wiring values into the generated `codecept.conf.js`.
|
|
9
|
+
|
|
10
|
+
This page is a reference for every `process.env.*` switch CodeceptJS reads or sets internally.
|
|
11
|
+
|
|
12
|
+
## Quick Reference
|
|
13
|
+
|
|
14
|
+
| Variable | Category | Default | Purpose |
|
|
15
|
+
| :--- | :--- | :--- | :--- |
|
|
16
|
+
| `profile` | Test execution | _unset_ | Profile name forwarded from `--profile`; readable from config and helpers. |
|
|
17
|
+
| `CI` | Test execution | _unset_ | Auto-set by every CI provider; enables CI-aware behavior. |
|
|
18
|
+
| `DONT_FAIL_ON_EMPTY_RUN` | Test execution | _unset_ | When set, do not fail CI if zero tests were executed. |
|
|
19
|
+
| `RUNS_WITH_WORKERS` | Workers | _unset_ | Set to `true` while running in workers; read by helpers/plugins to branch behavior. |
|
|
20
|
+
| `CODECEPT_WORKER_TIMEOUT` | Workers | `5m` | Hung-worker watchdog timeout. Accepts [ms](https://github.com/vercel/ms) strings (`30s`, `2m`, `1h`). |
|
|
21
|
+
| `CODECEPT_AUTO_EXIT_TIMEOUT` | Workers | `2000` | Time in ms allowed for helper cleanup before forced `process.exit`. Set to `0` to disable. |
|
|
22
|
+
| `DEBUG` | Debug | _unset_ | `debug` package namespace filter, e.g. `codeceptjs:*`, `codeceptjs:heal`. |
|
|
23
|
+
| `HEADLESS` | Init template | _unset_ | Read by `setHeadlessWhen()` in the generated config to toggle headless mode. |
|
|
24
|
+
| `BASE_URL` | Init template | `http://localhost` | Default URL written into the generated Playwright config. |
|
|
25
|
+
|
|
26
|
+
## Test Execution
|
|
27
|
+
|
|
28
|
+
### `profile`
|
|
29
|
+
|
|
30
|
+
Set automatically when you pass `--profile <name>` to `run`, `run-workers`, `run-rerun`, `run-multiple`, or `interactive`. The value is available inside `codecept.conf.js`, helpers, and plugins via `process.env.profile`, which is the standard way to switch configuration based on environment:
|
|
31
|
+
|
|
32
|
+
```js
|
|
33
|
+
exports.config = {
|
|
34
|
+
helpers: {
|
|
35
|
+
Playwright: {
|
|
36
|
+
url: process.env.profile === 'staging' ? 'https://staging.example.com' : 'http://localhost:3000',
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### `CI`
|
|
43
|
+
|
|
44
|
+
A de-facto standard variable exported by every major CI provider (GitHub Actions, GitLab CI, CircleCI, Jenkins, Travis, etc.). CodeceptJS uses it for two CI-aware behaviors:
|
|
45
|
+
|
|
46
|
+
- Extends the polling timeout for `submittedData()` to 60s (vs 2s locally).
|
|
47
|
+
- Fails the run with exit code `1` if no tests were executed (see `DONT_FAIL_ON_EMPTY_RUN`).
|
|
48
|
+
|
|
49
|
+
You should not need to set this manually.
|
|
50
|
+
|
|
51
|
+
### `DONT_FAIL_ON_EMPTY_RUN`
|
|
52
|
+
|
|
53
|
+
By default, when `CI` is set and no tests are executed (e.g. an over-restrictive `--grep`), CodeceptJS fails the run to surface false-positive green builds. Set `DONT_FAIL_ON_EMPTY_RUN=true` to keep the run green even when nothing ran.
|
|
54
|
+
|
|
55
|
+
```yaml
|
|
56
|
+
# .github/workflows/tests.yml
|
|
57
|
+
env:
|
|
58
|
+
DONT_FAIL_ON_EMPTY_RUN: 'true'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Workers
|
|
62
|
+
|
|
63
|
+
### `RUNS_WITH_WORKERS`
|
|
64
|
+
|
|
65
|
+
Automatically set to `'true'` while a `run-workers` invocation is active and reset to `'false'` when it finishes. Read it from helpers or plugins when you need to behave differently in worker mode — for example, to disable an interactive feature or change reporter output:
|
|
66
|
+
|
|
67
|
+
```js
|
|
68
|
+
if (process.env.RUNS_WITH_WORKERS === 'true') {
|
|
69
|
+
// running in a worker thread
|
|
70
|
+
}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Do not set this manually.
|
|
74
|
+
|
|
75
|
+
### `CODECEPT_WORKER_TIMEOUT`
|
|
76
|
+
|
|
77
|
+
Per-worker hung-process watchdog. If a worker produces no output for this duration, it is auto-terminated and reported as failed. Accepts any string parsed by the [`ms`](https://github.com/vercel/ms) package: `30s`, `2m`, `5m`, `1h`. Defaults to `5m`.
|
|
78
|
+
|
|
79
|
+
```sh
|
|
80
|
+
CODECEPT_WORKER_TIMEOUT=10m npx codeceptjs run-workers 4
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### `CODECEPT_AUTO_EXIT_TIMEOUT`
|
|
84
|
+
|
|
85
|
+
Time in **milliseconds** that CodeceptJS waits for helper `_cleanup()` hooks to complete before calling `process.exit()`. Defaults to `2000`. Set to `0` to disable the auto-exit and let the Node event loop drain naturally — useful when long-running async work (e.g. report uploads) must complete after the test run.
|
|
86
|
+
|
|
87
|
+
```sh
|
|
88
|
+
CODECEPT_AUTO_EXIT_TIMEOUT=0 npx codeceptjs run
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Debug
|
|
92
|
+
|
|
93
|
+
### `DEBUG`
|
|
94
|
+
|
|
95
|
+
Standard [`debug`](https://www.npmjs.com/package/debug) package selector. CodeceptJS emits debug output under the `codeceptjs:*` namespace. Useful namespaces:
|
|
96
|
+
|
|
97
|
+
| Namespace | Source |
|
|
98
|
+
| :--- | :--- |
|
|
99
|
+
| `codeceptjs:*` | All internal logs |
|
|
100
|
+
| `codeceptjs:recorder` | Recorder/promise queue |
|
|
101
|
+
| `codeceptjs:event` | Event dispatcher |
|
|
102
|
+
| `codeceptjs:container` | DI container |
|
|
103
|
+
| `codeceptjs:steps` | Step lifecycle |
|
|
104
|
+
| `codeceptjs:exit` | Exit listener |
|
|
105
|
+
| `codeceptjs:timeout` | Global timeout listener |
|
|
106
|
+
| `codeceptjs:pause` | Interactive pause |
|
|
107
|
+
| `codeceptjs:ai` | AI features |
|
|
108
|
+
| `codeceptjs:heal` | Heal plugin |
|
|
109
|
+
| `codeceptjs:analyze` | Analyze plugin |
|
|
110
|
+
| `codeceptjs:retryFailedStep` | retryFailedStep plugin |
|
|
111
|
+
| `codeceptjs:bdd` | Gherkin/BDD |
|
|
112
|
+
|
|
113
|
+
```sh
|
|
114
|
+
DEBUG="codeceptjs:*" npx codeceptjs run --verbose
|
|
115
|
+
DEBUG="codeceptjs:heal" npx codeceptjs run
|
|
116
|
+
DEBUG="codeceptjs:retryFailedStep" npx codeceptjs run
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
When `DEBUG` includes a `codeceptjs:` selector, the `heal` plugin keeps healing enabled in `--debug` mode (which otherwise disables it).
|
|
120
|
+
|
|
121
|
+
## Init Template
|
|
122
|
+
|
|
123
|
+
These variables are read by the configuration scaffolded by `codeceptjs init`. They have no effect unless your project's generated `codecept.conf.js` references them.
|
|
124
|
+
|
|
125
|
+
### `HEADLESS`
|
|
126
|
+
|
|
127
|
+
Read by [`setHeadlessWhen()`](https://www.npmjs.com/package/@codeceptjs/configure) in the generated config. When truthy, the browser starts headless. The init template wires this in so you can run `HEADLESS=true npx codeceptjs run` on CI without changing the config.
|
|
128
|
+
|
|
129
|
+
### `BASE_URL`
|
|
130
|
+
|
|
131
|
+
The init template uses `BASE_URL` (falling back to `http://localhost`) as the Playwright `url` value. After scaffolding, edit the generated config to change this default.
|
package/lib/codecept.js
CHANGED
|
@@ -316,7 +316,7 @@ class Codecept {
|
|
|
316
316
|
|
|
317
317
|
try {
|
|
318
318
|
event.emit(event.all.before, this)
|
|
319
|
-
mocha.run(async (failures) => await done(failures))
|
|
319
|
+
mocha.runner = mocha.run(async (failures) => await done(failures))
|
|
320
320
|
} catch (e) {
|
|
321
321
|
output.error(e.stack)
|
|
322
322
|
reject(e)
|
|
@@ -87,19 +87,5 @@ export default async function (workerCount, selectedRuns, options) {
|
|
|
87
87
|
process.exitCode = 1
|
|
88
88
|
} finally {
|
|
89
89
|
await workers.teardownAll()
|
|
90
|
-
|
|
91
|
-
// Force exit if event loop doesn't clear naturally
|
|
92
|
-
// This is needed because worker threads may leave handles open
|
|
93
|
-
// even after proper cleanup, preventing natural process termination
|
|
94
|
-
if (!options.noExit) {
|
|
95
|
-
// Use beforeExit to ensure we run after all other exit handlers
|
|
96
|
-
// have set the correct exit code
|
|
97
|
-
process.once('beforeExit', (code) => {
|
|
98
|
-
// Give cleanup a moment to complete, then force exit with the correct code
|
|
99
|
-
setTimeout(() => {
|
|
100
|
-
process.exit(code || process.exitCode || 0)
|
|
101
|
-
}, 100)
|
|
102
|
-
})
|
|
103
|
-
}
|
|
104
90
|
}
|
|
105
91
|
}
|
package/lib/command/run.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { getConfig, printError, getTestRoot, createOutputDir } from './utils.js'
|
|
1
|
+
import { getConfig, printError, getTestRoot, createOutputDir, autoExit } from './utils.js'
|
|
2
2
|
import Config from '../config.js'
|
|
3
3
|
import store from '../store.js'
|
|
4
4
|
import Codecept from '../codecept.js'
|
|
5
|
-
import container from '../container.js'
|
|
6
5
|
|
|
7
6
|
export default async function (test, options) {
|
|
8
7
|
// registering options globally to use in config
|
|
@@ -43,19 +42,6 @@ export default async function (test, options) {
|
|
|
43
42
|
process.exitCode = 1
|
|
44
43
|
} finally {
|
|
45
44
|
await codecept.teardown()
|
|
46
|
-
|
|
47
|
-
// Schedule a delayed exit to prevent process hanging due to browser helper event loops
|
|
48
|
-
// Only needed for Playwright/Puppeteer which keep the event loop alive
|
|
49
|
-
// Wait 1 second to allow final cleanup and output to complete
|
|
50
|
-
if (!process.env.CODECEPT_DISABLE_AUTO_EXIT) {
|
|
51
|
-
const helpers = container.helpers()
|
|
52
|
-
const hasBrowserHelper = helpers && (helpers.Playwright || helpers.Puppeteer || helpers.WebDriver)
|
|
53
|
-
|
|
54
|
-
if (hasBrowserHelper) {
|
|
55
|
-
setTimeout(() => {
|
|
56
|
-
process.exit(process.exitCode || 0)
|
|
57
|
-
}, 1000).unref()
|
|
58
|
-
}
|
|
59
|
-
}
|
|
45
|
+
await autoExit()
|
|
60
46
|
}
|
|
61
47
|
}
|
package/lib/command/utils.js
CHANGED
|
@@ -107,6 +107,20 @@ export const createOutputDir = (config, testRoot) => {
|
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
export async function autoExit() {
|
|
111
|
+
const timeout = parseInt(process.env.CODECEPT_AUTO_EXIT_TIMEOUT, 10)
|
|
112
|
+
if (timeout === 0) return
|
|
113
|
+
const exitTimeout = timeout || 2000
|
|
114
|
+
|
|
115
|
+
const { default: container } = await import('../container.js')
|
|
116
|
+
const helpers = container.helpers()
|
|
117
|
+
if (!helpers || !Object.values(helpers).some(h => typeof h._cleanup === 'function')) return
|
|
118
|
+
|
|
119
|
+
const { default: recorder } = await import('../recorder.js')
|
|
120
|
+
await Promise.race([recorder.promise(), new Promise(resolve => setTimeout(resolve, exitTimeout))])
|
|
121
|
+
process.exit(process.exitCode || 0)
|
|
122
|
+
}
|
|
123
|
+
|
|
110
124
|
export const findConfigFile = testsPath => {
|
|
111
125
|
const extensions = ['js', 'ts']
|
|
112
126
|
for (const ext of extensions) {
|
package/lib/heal.js
CHANGED
|
@@ -54,7 +54,9 @@ class Heal {
|
|
|
54
54
|
|
|
55
55
|
async getCodeSuggestions(context) {
|
|
56
56
|
const suggestions = []
|
|
57
|
+
const stepName = context.step?.name
|
|
57
58
|
const recipes = matchRecipes(this.recipes, this.contextName)
|
|
59
|
+
.filter(r => !r.steps || !stepName || r.steps.includes(stepName))
|
|
58
60
|
|
|
59
61
|
debug('Recipes', recipes)
|
|
60
62
|
|
package/lib/helper/Playwright.js
CHANGED
|
@@ -755,6 +755,11 @@ class Playwright extends Helper {
|
|
|
755
755
|
}
|
|
756
756
|
|
|
757
757
|
async _afterSuite() {
|
|
758
|
+
// Reset leftover test-level cleanup state (e.g. screenshot failures)
|
|
759
|
+
// so only errors from this suite teardown are evaluated below.
|
|
760
|
+
this.hasCleanupError = false
|
|
761
|
+
this.testFailures = []
|
|
762
|
+
|
|
758
763
|
// Stop browser after suite completes
|
|
759
764
|
// For restart strategies: stop after each suite
|
|
760
765
|
// For session mode (restart:false): stop after the last suite
|
package/lib/plugin/aiTrace.js
CHANGED
|
@@ -92,6 +92,7 @@ export default function (config = {}) {
|
|
|
92
92
|
let testStartTime
|
|
93
93
|
let currentUrl = null
|
|
94
94
|
let testFailed = false
|
|
95
|
+
let pendingArtifactCapture = null
|
|
95
96
|
let firstFailedStepSaved = false
|
|
96
97
|
|
|
97
98
|
const reportDir = config.output ? path.resolve(store.codeceptDir, config.output) : defaultConfig.output
|
|
@@ -129,6 +130,7 @@ export default function (config = {}) {
|
|
|
129
130
|
currentUrl = null
|
|
130
131
|
testFailed = false
|
|
131
132
|
firstFailedStepSaved = false
|
|
133
|
+
pendingArtifactCapture = null
|
|
132
134
|
})
|
|
133
135
|
|
|
134
136
|
event.dispatcher.on(event.step.after, step => {
|
|
@@ -162,13 +164,12 @@ export default function (config = {}) {
|
|
|
162
164
|
return
|
|
163
165
|
}
|
|
164
166
|
|
|
165
|
-
|
|
167
|
+
recorder.add(`aiTrace step persistence: ${step.toString()}`, () => persistStep(step).catch(err => {
|
|
166
168
|
output.debug(`aiTrace: Error saving step: ${err.message}`)
|
|
167
|
-
})
|
|
168
|
-
recorder.add(`wait aiTrace step persistence: ${step.toString()}`, () => stepPersistPromise, true)
|
|
169
|
+
}), true)
|
|
169
170
|
})
|
|
170
171
|
|
|
171
|
-
event.dispatcher.on(event.step.failed,
|
|
172
|
+
event.dispatcher.on(event.step.failed, step => {
|
|
172
173
|
if (!currentTest) return
|
|
173
174
|
if (step.status === 'queued' && testFailed) {
|
|
174
175
|
output.debug(`aiTrace: Skipping queued failed step "${step.toString()}" - testFailed: ${testFailed}`)
|
|
@@ -188,11 +189,9 @@ export default function (config = {}) {
|
|
|
188
189
|
}
|
|
189
190
|
existingStep.status = 'failed'
|
|
190
191
|
|
|
191
|
-
|
|
192
|
-
await captureArtifactsForStep(step, existingStep, existingStep.prefix)
|
|
193
|
-
} catch (err) {
|
|
192
|
+
pendingArtifactCapture = captureArtifactsForStep(step, existingStep, existingStep.prefix).catch(err => {
|
|
194
193
|
output.debug(`aiTrace: Error updating failed step: ${err.message}`)
|
|
195
|
-
}
|
|
194
|
+
})
|
|
196
195
|
} else {
|
|
197
196
|
if (stepNum === -1) return
|
|
198
197
|
if (isStepIgnored(step)) return
|
|
@@ -218,11 +217,9 @@ export default function (config = {}) {
|
|
|
218
217
|
steps.push(stepData)
|
|
219
218
|
firstFailedStepSaved = true
|
|
220
219
|
|
|
221
|
-
|
|
222
|
-
await captureArtifactsForStep(step, stepData, stepPrefix)
|
|
223
|
-
} catch (err) {
|
|
220
|
+
pendingArtifactCapture = captureArtifactsForStep(step, stepData, stepPrefix).catch(err => {
|
|
224
221
|
output.debug(`aiTrace: Error capturing failed step artifacts: ${err.message}`)
|
|
225
|
-
}
|
|
222
|
+
})
|
|
226
223
|
}
|
|
227
224
|
})
|
|
228
225
|
|
|
@@ -238,7 +235,13 @@ export default function (config = {}) {
|
|
|
238
235
|
if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') {
|
|
239
236
|
return
|
|
240
237
|
}
|
|
241
|
-
persist
|
|
238
|
+
recorder.add('aiTrace:persist failed', async () => {
|
|
239
|
+
if (pendingArtifactCapture) {
|
|
240
|
+
await pendingArtifactCapture
|
|
241
|
+
pendingArtifactCapture = null
|
|
242
|
+
}
|
|
243
|
+
persist(test, 'failed')
|
|
244
|
+
}, true)
|
|
242
245
|
})
|
|
243
246
|
|
|
244
247
|
async function persistStep(step) {
|
package/lib/plugin/analyze.js
CHANGED
|
@@ -12,6 +12,7 @@ const ai = aiModule.default || aiModule
|
|
|
12
12
|
import colors from 'chalk'
|
|
13
13
|
import ora from 'ora'
|
|
14
14
|
import event from '../event.js'
|
|
15
|
+
import recorder from '../recorder.js'
|
|
15
16
|
|
|
16
17
|
import output from '../output.js'
|
|
17
18
|
|
|
@@ -227,14 +228,14 @@ export default function (config = {}) {
|
|
|
227
228
|
console.log('Enabled AI analysis')
|
|
228
229
|
})
|
|
229
230
|
|
|
230
|
-
event.dispatcher.on(event.all.result,
|
|
231
|
+
event.dispatcher.on(event.all.result, result => {
|
|
231
232
|
if (!isMainThread) return // run only on main thread
|
|
232
233
|
if (!ai.isEnabled) {
|
|
233
234
|
console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
|
|
234
235
|
return
|
|
235
236
|
}
|
|
236
237
|
|
|
237
|
-
printReport(result)
|
|
238
|
+
recorder.add('analyze:print-ai-report', () => printReport(result), true)
|
|
238
239
|
})
|
|
239
240
|
|
|
240
241
|
event.dispatcher.on(event.workers.result, async result => {
|
|
@@ -248,7 +249,7 @@ export default function (config = {}) {
|
|
|
248
249
|
return
|
|
249
250
|
}
|
|
250
251
|
|
|
251
|
-
printReport(result)
|
|
252
|
+
await printReport(result)
|
|
252
253
|
})
|
|
253
254
|
|
|
254
255
|
async function printReport(result) {
|
|
@@ -294,7 +295,7 @@ export default function (config = {}) {
|
|
|
294
295
|
console.error('Error analyzing failed tests', err)
|
|
295
296
|
}
|
|
296
297
|
|
|
297
|
-
if (!Object.keys(
|
|
298
|
+
if (!Object.keys(Container.plugins()).includes('pageInfo')) {
|
|
298
299
|
console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
|
|
299
300
|
}
|
|
300
301
|
}
|
package/lib/plugin/heal.js
CHANGED
|
@@ -80,6 +80,7 @@ export default function (config = {}) {
|
|
|
80
80
|
event.dispatcher.on(event.test.before, test => {
|
|
81
81
|
currentTest = test
|
|
82
82
|
healedSteps = 0
|
|
83
|
+
healTries = 0
|
|
83
84
|
caughtError = null
|
|
84
85
|
})
|
|
85
86
|
|
|
@@ -94,7 +95,9 @@ export default function (config = {}) {
|
|
|
94
95
|
if (trigger.on === 'file' && !matchStepFile(step, trigger.path, trigger.line)) return
|
|
95
96
|
|
|
96
97
|
recorder.catchWithoutStop(async err => {
|
|
98
|
+
if (healTries >= config.healLimit) throw err
|
|
97
99
|
isHealing = true
|
|
100
|
+
healTries++
|
|
98
101
|
if (caughtError === err) throw err // avoid double handling
|
|
99
102
|
caughtError = err
|
|
100
103
|
|
|
@@ -121,8 +124,6 @@ export default function (config = {}) {
|
|
|
121
124
|
|
|
122
125
|
await heal.healStep(step, err, { test })
|
|
123
126
|
|
|
124
|
-
healTries++
|
|
125
|
-
|
|
126
127
|
recorder.add('close healing session', () => {
|
|
127
128
|
recorder.reset()
|
|
128
129
|
recorder.session.restore('heal')
|
package/lib/plugin/pageInfo.js
CHANGED
|
@@ -40,10 +40,10 @@ const defaultConfig = {
|
|
|
40
40
|
export default function (config = {}) {
|
|
41
41
|
config = Object.assign(defaultConfig, config)
|
|
42
42
|
|
|
43
|
-
const helper = pickActingHelper(Container.helpers())
|
|
44
|
-
if (!helper) return
|
|
45
|
-
|
|
46
43
|
event.dispatcher.on(event.test.failed, test => {
|
|
44
|
+
const helper = pickActingHelper(Container.helpers())
|
|
45
|
+
if (!helper) return
|
|
46
|
+
|
|
47
47
|
const pageState = {}
|
|
48
48
|
|
|
49
49
|
recorder.add('pageInfo capture', async () => {
|
|
@@ -60,8 +60,6 @@ export default function (config = {}) {
|
|
|
60
60
|
if (captured.html) {
|
|
61
61
|
const htmlPath = path.join(store.outputDir, captured.html)
|
|
62
62
|
pageState.htmlSnapshot = htmlPath
|
|
63
|
-
// Scan raw HTML (pre-cleanHtml) so error classes containing digits
|
|
64
|
-
// or trash-class prefixes aren't stripped before detection.
|
|
65
63
|
const htmlForScan = captured.htmlRaw || (() => {
|
|
66
64
|
try { return fs.readFileSync(htmlPath, 'utf8') } catch { return '' }
|
|
67
65
|
})()
|
|
@@ -90,7 +88,7 @@ export default function (config = {}) {
|
|
|
90
88
|
} catch {}
|
|
91
89
|
}
|
|
92
90
|
} catch {}
|
|
93
|
-
})
|
|
91
|
+
}, true)
|
|
94
92
|
|
|
95
93
|
recorder.add('Save page info', () => {
|
|
96
94
|
test.addNote('pageInfo', pageStateToMarkdown(pageState))
|
|
@@ -99,7 +97,7 @@ export default function (config = {}) {
|
|
|
99
97
|
fs.writeFileSync(pageStateFileName, pageStateToMarkdown(pageState))
|
|
100
98
|
test.artifacts.pageInfo = pageStateFileName
|
|
101
99
|
return pageState
|
|
102
|
-
})
|
|
100
|
+
}, true)
|
|
103
101
|
})
|
|
104
102
|
}
|
|
105
103
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import debugModule from 'debug'
|
|
1
2
|
import event from '../event.js'
|
|
2
3
|
import recorder from '../recorder.js'
|
|
3
4
|
import store from '../store.js'
|
|
4
5
|
|
|
6
|
+
const debug = debugModule('codeceptjs:retryFailedStep')
|
|
7
|
+
|
|
5
8
|
const defaultConfig = {
|
|
6
9
|
retries: 3,
|
|
7
10
|
defaultIgnoredSteps: ['amOnPage', 'wait*', 'send*', 'execute*', 'run*', 'have*'],
|
|
@@ -147,9 +150,7 @@ export default function (config) {
|
|
|
147
150
|
test.opts.conditionalRetries = config.retries
|
|
148
151
|
test.opts.stepRetryPriority = stepRetryPriority
|
|
149
152
|
|
|
150
|
-
|
|
151
|
-
console.log('[retryFailedStep] applying retries =', config.retries, 'for test', test.title)
|
|
152
|
-
}
|
|
153
|
+
debug('applying retries = %d for test %s', config.retries, test.title)
|
|
153
154
|
recorder.retry(config)
|
|
154
155
|
})
|
|
155
156
|
|
package/lib/plugin/screencast.js
CHANGED
|
@@ -106,6 +106,12 @@ function wireScreencast(mode, options) {
|
|
|
106
106
|
state.startedAt = options.subtitles ? Date.now() : null
|
|
107
107
|
})
|
|
108
108
|
|
|
109
|
+
event.dispatcher.on(event.test.started, test => {
|
|
110
|
+
if (!options.video || state.startQueued) return
|
|
111
|
+
state.startQueued = true
|
|
112
|
+
recorder.add('screencast:start', async () => startScreencast(state.test, options, state), true)
|
|
113
|
+
})
|
|
114
|
+
|
|
109
115
|
event.dispatcher.on(event.step.started, step => {
|
|
110
116
|
if (state.steps) {
|
|
111
117
|
const at = Date.now()
|
|
@@ -116,10 +122,6 @@ function wireScreencast(mode, options) {
|
|
|
116
122
|
title: stepTitle(step),
|
|
117
123
|
}
|
|
118
124
|
}
|
|
119
|
-
if (!options.video || state.startQueued || !state.test) return
|
|
120
|
-
state.startQueued = true
|
|
121
|
-
const test = state.test
|
|
122
|
-
recorder.add('screencast:start', async () => startScreencast(test, options, state), true)
|
|
123
125
|
})
|
|
124
126
|
|
|
125
127
|
if (options.subtitles) {
|
package/lib/workers.js
CHANGED
|
@@ -547,10 +547,11 @@ class Workers extends EventEmitter {
|
|
|
547
547
|
if (this.isPoolMode) {
|
|
548
548
|
this.activeWorkers.set(worker, { available: true, workerIndex: null })
|
|
549
549
|
}
|
|
550
|
-
|
|
550
|
+
|
|
551
551
|
// Track last activity time to detect hanging workers
|
|
552
552
|
let lastActivity = Date.now()
|
|
553
553
|
let currentTest = null
|
|
554
|
+
let autoTerminated = false
|
|
554
555
|
const workerTimeout = process.env.CODECEPT_WORKER_TIMEOUT ? ms(process.env.CODECEPT_WORKER_TIMEOUT) : ms('5m')
|
|
555
556
|
|
|
556
557
|
const timeoutChecker = setInterval(() => {
|
|
@@ -611,6 +612,13 @@ class Workers extends EventEmitter {
|
|
|
611
612
|
})
|
|
612
613
|
}
|
|
613
614
|
|
|
615
|
+
const exitTimeout = parseInt(process.env.CODECEPT_AUTO_EXIT_TIMEOUT, 10)
|
|
616
|
+
if (exitTimeout === 0) break
|
|
617
|
+
setTimeout(() => {
|
|
618
|
+
autoTerminated = true
|
|
619
|
+
worker.terminate()
|
|
620
|
+
}, exitTimeout || 2000)
|
|
621
|
+
|
|
614
622
|
break
|
|
615
623
|
case event.suite.before:
|
|
616
624
|
{
|
|
@@ -741,8 +749,8 @@ class Workers extends EventEmitter {
|
|
|
741
749
|
worker.on('exit', (code) => {
|
|
742
750
|
clearInterval(timeoutChecker)
|
|
743
751
|
this.closedWorkers += 1
|
|
744
|
-
|
|
745
|
-
if (code !== 0) {
|
|
752
|
+
|
|
753
|
+
if (code !== 0 && !autoTerminated) {
|
|
746
754
|
console.error(`[Main] Worker exited with code ${code}`)
|
|
747
755
|
if (currentTest) {
|
|
748
756
|
console.error(`[Main] Last test running: ${currentTest}`)
|