@flakiness/cucumberjs 1.0.0-alpha.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +201 -0
- package/lib/formatter.js +448 -0
- package/package.json +12 -7
- package/types/src/formatter.d.ts +23 -0
- package/types/src/formatter.d.ts.map +1 -0
- package/.github/workflows/flakiness-upload-fork-prs.yml +0 -30
- package/.github/workflows/publish-npm.yml +0 -41
- package/.github/workflows/tests.yml +0 -45
- package/CONTRIBUTING.md +0 -58
- package/agenda.md +0 -2
- package/build.mts +0 -34
- package/cucumber.mjs +0 -6
- package/features/attachments.feature +0 -32
- package/features/basic.feature +0 -27
- package/features/data_tables.feature +0 -45
- package/features/description.feature +0 -49
- package/features/errors.feature +0 -28
- package/features/hooks_named.feature +0 -32
- package/features/hooks_unnamed.feature +0 -33
- package/features/locations.feature +0 -37
- package/features/retries.feature +0 -30
- package/features/rules.feature +0 -25
- package/features/scenario_outlines.feature +0 -57
- package/features/scenario_outlines_multiple.feature +0 -44
- package/features/statuses.feature +0 -70
- package/features/stdio.feature +0 -29
- package/features/steps.feature +0 -24
- package/features/support/attachments_steps.ts +0 -32
- package/features/support/basic_steps.ts +0 -235
- package/features/support/description_steps.ts +0 -37
- package/features/support/errors_steps.ts +0 -48
- package/features/support/harness.ts +0 -196
- package/features/support/project_steps.ts +0 -24
- package/features/support/stdio_steps.ts +0 -21
- package/features/support/tags_steps.ts +0 -10
- package/features/tags.feature +0 -19
- package/features/tags_hierarchy.feature +0 -37
- package/plan.md +0 -59
- package/pnpm-workspace.yaml +0 -2
- package/src/formatter.ts +0 -635
- package/tsconfig.json +0 -24
package/README.md
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
[](https://flakiness.io/flakiness/cucumberjs)
|
|
2
|
+
|
|
3
|
+
# Flakiness.io CucumberJS Formatter
|
|
4
|
+
|
|
5
|
+
A custom CucumberJS formatter that generates Flakiness Reports from your Cucumber test runs. The formatter automatically converts CucumberJS test results into the standardized [Flakiness JSON format](https://github.com/flakiness/flakiness-report), preserving complete Gherkin structure, test outcomes, and environment information.
|
|
6
|
+
|
|
7
|
+
## Supported Gherkin Features
|
|
8
|
+
|
|
9
|
+
- Scenarios & Scenario Outlines (with multiple Examples blocks)
|
|
10
|
+
- Rules
|
|
11
|
+
- Tags & tag inheritance (Feature → Rule → Scenario → Examples)
|
|
12
|
+
- Steps (Given/When/Then with keyword prefix)
|
|
13
|
+
- Data Tables
|
|
14
|
+
- Before & After Hooks (named and unnamed)
|
|
15
|
+
- Feature, Rule, and Scenario descriptions
|
|
16
|
+
- Attachments (`this.attach()` and `this.log()`)
|
|
17
|
+
- Retries (`--retry`)
|
|
18
|
+
- Parallel execution (`--parallel`)
|
|
19
|
+
- All statuses: passed, failed, pending, undefined, ambiguous, skipped
|
|
20
|
+
|
|
21
|
+
## Table of Contents
|
|
22
|
+
|
|
23
|
+
- [Requirements](#requirements)
|
|
24
|
+
- [Installation](#installation)
|
|
25
|
+
- [Quick Start](#quick-start)
|
|
26
|
+
- [Uploading Reports](#uploading-reports)
|
|
27
|
+
- [Viewing Reports](#viewing-reports)
|
|
28
|
+
- [Features](#features)
|
|
29
|
+
- [Environment Detection](#environment-detection)
|
|
30
|
+
- [CI Integration](#ci-integration)
|
|
31
|
+
- [Configuration Options](#configuration-options)
|
|
32
|
+
- [`flakinessProject?: string`](#flakinessproject-string)
|
|
33
|
+
- [`endpoint?: string`](#endpoint-string)
|
|
34
|
+
- [`token?: string`](#token-string)
|
|
35
|
+
- [`outputFolder?: string`](#outputfolder-string)
|
|
36
|
+
- [`disableUpload?: boolean`](#disableupload-boolean)
|
|
37
|
+
- [Example Configuration](#example-configuration)
|
|
38
|
+
|
|
39
|
+
## Requirements
|
|
40
|
+
|
|
41
|
+
- `@cucumber/cucumber` 12.0 or higher
|
|
42
|
+
- Node.js project with a git repository (for commit information)
|
|
43
|
+
|
|
44
|
+
## Installation
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
npm install -D @flakiness/cucumberjs
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
Add the formatter to your `cucumber.mjs`:
|
|
53
|
+
|
|
54
|
+
```javascript
|
|
55
|
+
export default {
|
|
56
|
+
paths: ['features/**/*.feature'],
|
|
57
|
+
import: ['features/support/**/*.ts'],
|
|
58
|
+
format: ['@flakiness/cucumberjs', 'progress'],
|
|
59
|
+
formatOptions: {
|
|
60
|
+
flakinessProject: 'my-org/my-project',
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Run your tests. The report will be automatically generated in the `./flakiness-report` folder:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
npx cucumber-js
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
View the interactive report:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
npx flakiness show ./flakiness-report
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Uploading Reports
|
|
78
|
+
|
|
79
|
+
Reports are automatically uploaded to Flakiness.io after test completion. Authentication can be done in two ways:
|
|
80
|
+
|
|
81
|
+
- **Access token**: Provide a token via the `token` format option or the `FLAKINESS_ACCESS_TOKEN` environment variable.
|
|
82
|
+
- **GitHub OIDC**: When running in GitHub Actions, the formatter can authenticate using GitHub's OIDC token — no access token needed. See [GitHub Actions integration](https://docs.flakiness.io/ci/github-actions/) for setup instructions.
|
|
83
|
+
|
|
84
|
+
If upload fails, the report is still available locally in the output folder.
|
|
85
|
+
|
|
86
|
+
## Viewing Reports
|
|
87
|
+
|
|
88
|
+
After test execution, you can view the report using:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
npx flakiness show ./flakiness-report
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Features
|
|
95
|
+
|
|
96
|
+
### Environment Detection
|
|
97
|
+
|
|
98
|
+
Environment variables prefixed with `FK_ENV_` are automatically included in the environment metadata. The prefix is stripped and the key is converted to lowercase.
|
|
99
|
+
|
|
100
|
+
**Example:**
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
export FK_ENV_DEPLOYMENT=staging
|
|
104
|
+
export FK_ENV_REGION=us-east-1
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
This will result in the environment containing:
|
|
108
|
+
```json
|
|
109
|
+
{
|
|
110
|
+
"metadata": {
|
|
111
|
+
"deployment": "staging",
|
|
112
|
+
"region": "us-east-1"
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Flakiness.io will create a dedicated history for tests executed in each unique environment. This means tests run with `FK_ENV_DEPLOYMENT=staging` will have a separate timeline from tests run with `FK_ENV_DEPLOYMENT=production`, allowing you to track flakiness patterns specific to each deployment environment.
|
|
118
|
+
|
|
119
|
+
### CI Integration
|
|
120
|
+
|
|
121
|
+
The formatter automatically detects CI environments and includes:
|
|
122
|
+
- CI run URLs (GitHub Actions, Azure DevOps, Jenkins, GitLab CI)
|
|
123
|
+
- Git commit information
|
|
124
|
+
- System environment data
|
|
125
|
+
|
|
126
|
+
## Configuration Options
|
|
127
|
+
|
|
128
|
+
All options are passed via CucumberJS's `formatOptions` in your configuration file.
|
|
129
|
+
|
|
130
|
+
### `flakinessProject?: string`
|
|
131
|
+
|
|
132
|
+
The Flakiness.io project identifier in `org/project` format. Used for GitHub OIDC authentication — when set, and the Flakiness.io project is bound to the GitHub repository running the workflow, the formatter authenticates uploads via GitHub Actions OIDC token with no access token required.
|
|
133
|
+
|
|
134
|
+
```javascript
|
|
135
|
+
formatOptions: {
|
|
136
|
+
flakinessProject: 'my-org/my-project',
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### `endpoint?: string`
|
|
141
|
+
|
|
142
|
+
Custom Flakiness.io endpoint URL for uploading reports. Defaults to the `FLAKINESS_ENDPOINT` environment variable, or `https://flakiness.io` if not set.
|
|
143
|
+
|
|
144
|
+
Use this option to point to a custom or self-hosted Flakiness.io instance.
|
|
145
|
+
|
|
146
|
+
```javascript
|
|
147
|
+
formatOptions: {
|
|
148
|
+
endpoint: 'https://custom.flakiness.io',
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### `token?: string`
|
|
153
|
+
|
|
154
|
+
Access token for authenticating with Flakiness.io when uploading reports. Defaults to the `FLAKINESS_ACCESS_TOKEN` environment variable.
|
|
155
|
+
|
|
156
|
+
If no token is provided, the formatter will attempt to authenticate using GitHub OIDC.
|
|
157
|
+
|
|
158
|
+
```javascript
|
|
159
|
+
formatOptions: {
|
|
160
|
+
token: 'your-access-token',
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### `outputFolder?: string`
|
|
165
|
+
|
|
166
|
+
Directory path where the Flakiness report will be written. Defaults to `flakiness-report` in the current working directory, or the `FLAKINESS_OUTPUT_DIR` environment variable if set.
|
|
167
|
+
|
|
168
|
+
```javascript
|
|
169
|
+
formatOptions: {
|
|
170
|
+
outputFolder: './test-results/flakiness',
|
|
171
|
+
}
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### `disableUpload?: boolean`
|
|
175
|
+
|
|
176
|
+
When set to `true`, prevents uploading the report to Flakiness.io. The report is still generated locally. Can also be controlled via the `FLAKINESS_DISABLE_UPLOAD` environment variable.
|
|
177
|
+
|
|
178
|
+
```javascript
|
|
179
|
+
formatOptions: {
|
|
180
|
+
disableUpload: true,
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
## Example Configuration
|
|
185
|
+
|
|
186
|
+
Here's a complete example with all options:
|
|
187
|
+
|
|
188
|
+
```javascript
|
|
189
|
+
export default {
|
|
190
|
+
paths: ['features/**/*.feature'],
|
|
191
|
+
import: ['features/support/**/*.ts'],
|
|
192
|
+
format: ['@flakiness/cucumberjs', 'progress'],
|
|
193
|
+
formatOptions: {
|
|
194
|
+
flakinessProject: 'my-org/my-project',
|
|
195
|
+
endpoint: process.env.FLAKINESS_ENDPOINT,
|
|
196
|
+
token: process.env.FLAKINESS_ACCESS_TOKEN,
|
|
197
|
+
outputFolder: './flakiness-report',
|
|
198
|
+
disableUpload: false,
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
```
|
package/lib/formatter.js
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import { Formatter, formatterHelpers } from "@cucumber/cucumber";
|
|
2
|
+
import { AttachmentContentEncoding, TestStepResultStatus } from "@cucumber/messages";
|
|
3
|
+
import {
|
|
4
|
+
CIUtils,
|
|
5
|
+
CPUUtilization,
|
|
6
|
+
GitWorktree,
|
|
7
|
+
RAMUtilization,
|
|
8
|
+
ReportUtils,
|
|
9
|
+
uploadReport,
|
|
10
|
+
writeReport
|
|
11
|
+
} from "@flakiness/sdk";
|
|
12
|
+
import fs from "node:fs";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
const CUCUMBER_LOG_MEDIA_TYPE = "text/x.cucumber.log+plain";
|
|
15
|
+
class FlakinessCucumberFormatter extends Formatter {
|
|
16
|
+
static documentation = "Generates a Flakiness report for a CucumberJS run.";
|
|
17
|
+
_config;
|
|
18
|
+
_cpuUtilization = new CPUUtilization({ precision: 10 });
|
|
19
|
+
_ramUtilization = new RAMUtilization({ precision: 10 });
|
|
20
|
+
_startTimestamp = Date.now();
|
|
21
|
+
_outputFolder;
|
|
22
|
+
_telemetryTimer;
|
|
23
|
+
_finishedPromise = new ManualPromise();
|
|
24
|
+
_testCaseStartedById = /* @__PURE__ */ new Map();
|
|
25
|
+
_testCaseFinishedById = /* @__PURE__ */ new Map();
|
|
26
|
+
constructor(options) {
|
|
27
|
+
super(options);
|
|
28
|
+
this._config = parseFormatterConfig(options.parsedArgvOptions);
|
|
29
|
+
this._outputFolder = path.resolve(this.cwd, this._config.outputFolder ?? process.env.FLAKINESS_OUTPUT_DIR ?? "flakiness-report");
|
|
30
|
+
this._sampleSystem = this._sampleSystem.bind(this);
|
|
31
|
+
this._sampleSystem();
|
|
32
|
+
options.eventBroadcaster.on("envelope", (envelope) => {
|
|
33
|
+
if (envelope.testRunStarted)
|
|
34
|
+
this._onTestRunStarted(envelope.testRunStarted);
|
|
35
|
+
if (envelope.testCaseStarted)
|
|
36
|
+
this._onTestCaseStarted(envelope.testCaseStarted);
|
|
37
|
+
if (envelope.testCaseFinished)
|
|
38
|
+
this._onTestCaseFinished(envelope.testCaseFinished);
|
|
39
|
+
if (envelope.testRunFinished) {
|
|
40
|
+
this._onTestRunFinished(envelope.testRunFinished).then(() => this._finishedPromise.resolve(void 0)).catch((e) => this._finishedPromise.reject(e));
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
_onTestRunStarted(testRunStarted) {
|
|
45
|
+
this._startTimestamp = toUnixTimestampMS(testRunStarted.timestamp);
|
|
46
|
+
}
|
|
47
|
+
_onTestCaseStarted(testCaseStarted) {
|
|
48
|
+
this._testCaseStartedById.set(testCaseStarted.id, testCaseStarted);
|
|
49
|
+
}
|
|
50
|
+
_onTestCaseFinished(testCaseFinished) {
|
|
51
|
+
this._testCaseFinishedById.set(testCaseFinished.testCaseStartedId, testCaseFinished);
|
|
52
|
+
}
|
|
53
|
+
async finished() {
|
|
54
|
+
if (this._telemetryTimer)
|
|
55
|
+
clearTimeout(this._telemetryTimer);
|
|
56
|
+
try {
|
|
57
|
+
await this._finishedPromise.promise;
|
|
58
|
+
} catch (error) {
|
|
59
|
+
console.error(`[flakiness.io] Failed to generate report: ${error instanceof Error ? error.stack ?? error.message : String(error)}`);
|
|
60
|
+
}
|
|
61
|
+
await super.finished();
|
|
62
|
+
}
|
|
63
|
+
_sampleSystem() {
|
|
64
|
+
this._cpuUtilization.sample();
|
|
65
|
+
this._ramUtilization.sample();
|
|
66
|
+
this._telemetryTimer = setTimeout(this._sampleSystem, 1e3);
|
|
67
|
+
}
|
|
68
|
+
async _onTestRunFinished(testRunFinished) {
|
|
69
|
+
this._cpuUtilization.sample();
|
|
70
|
+
this._ramUtilization.sample();
|
|
71
|
+
let worktree;
|
|
72
|
+
let commitId;
|
|
73
|
+
try {
|
|
74
|
+
worktree = GitWorktree.create(this.cwd);
|
|
75
|
+
commitId = worktree.headCommitId();
|
|
76
|
+
} catch {
|
|
77
|
+
console.warn("[flakiness.io] Failed to fetch commit info - is this a git repo?");
|
|
78
|
+
console.error("[flakiness.io] Report is NOT generated.");
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const { attachments, suites } = await this._collectSuites(worktree);
|
|
82
|
+
const report = ReportUtils.normalizeReport({
|
|
83
|
+
category: "cucumberjs",
|
|
84
|
+
commitId,
|
|
85
|
+
duration: toUnixTimestampMS(testRunFinished.timestamp) - this._startTimestamp,
|
|
86
|
+
environments: [
|
|
87
|
+
ReportUtils.createEnvironment({
|
|
88
|
+
name: "cucumberjs"
|
|
89
|
+
})
|
|
90
|
+
],
|
|
91
|
+
flakinessProject: this._config.flakinessProject,
|
|
92
|
+
suites,
|
|
93
|
+
startTimestamp: this._startTimestamp,
|
|
94
|
+
url: CIUtils.runUrl()
|
|
95
|
+
});
|
|
96
|
+
ReportUtils.collectSources(worktree, report);
|
|
97
|
+
this._cpuUtilization.enrich(report);
|
|
98
|
+
this._ramUtilization.enrich(report);
|
|
99
|
+
await writeReport(report, attachments, this._outputFolder);
|
|
100
|
+
const disableUpload = this._config.disableUpload ?? envBool("FLAKINESS_DISABLE_UPLOAD");
|
|
101
|
+
if (!disableUpload) {
|
|
102
|
+
await uploadReport(report, attachments, {
|
|
103
|
+
flakinessAccessToken: this._config.token,
|
|
104
|
+
flakinessEndpoint: this._config.endpoint
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
const defaultOutputFolder = path.join(this.cwd, "flakiness-report");
|
|
108
|
+
const folder = defaultOutputFolder === this._outputFolder ? "" : path.relative(this.cwd, this._outputFolder);
|
|
109
|
+
this.log(`
|
|
110
|
+
To open last Flakiness report, run:
|
|
111
|
+
|
|
112
|
+
npx flakiness show ${folder}
|
|
113
|
+
`);
|
|
114
|
+
}
|
|
115
|
+
async _collectSuites(worktree) {
|
|
116
|
+
const suitesByKey = /* @__PURE__ */ new Map();
|
|
117
|
+
const testsById = /* @__PURE__ */ new Map();
|
|
118
|
+
const attachments = /* @__PURE__ */ new Map();
|
|
119
|
+
const parallelIndexByWorkerId = /* @__PURE__ */ new Map();
|
|
120
|
+
for (const [testCaseStartedId, testCaseStarted] of this._testCaseStartedById) {
|
|
121
|
+
const attemptData = this.eventDataCollector.getTestCaseAttempt(testCaseStartedId);
|
|
122
|
+
const parsedAttempt = formatterHelpers.parseTestCaseAttempt({
|
|
123
|
+
testCaseAttempt: attemptData,
|
|
124
|
+
snippetBuilder: this.snippetBuilder,
|
|
125
|
+
supportCodeLibrary: this.supportCodeLibrary
|
|
126
|
+
});
|
|
127
|
+
const featureUri = attemptData.pickle.uri;
|
|
128
|
+
const fileSuite = getOrCreateFileSuite(suitesByKey, worktree, this.cwd, featureUri);
|
|
129
|
+
const featureSuite = getOrCreateFeatureSuite(
|
|
130
|
+
suitesByKey,
|
|
131
|
+
fileSuite,
|
|
132
|
+
worktree,
|
|
133
|
+
this.cwd,
|
|
134
|
+
featureUri,
|
|
135
|
+
attemptData.gherkinDocument
|
|
136
|
+
);
|
|
137
|
+
const rule = findRuleForPickle(attemptData.gherkinDocument, attemptData.pickle);
|
|
138
|
+
const parentSuite = rule ? getOrCreateRuleSuite(suitesByKey, featureSuite, worktree, this.cwd, featureUri, rule) : featureSuite;
|
|
139
|
+
let test = testsById.get(attemptData.testCase.id);
|
|
140
|
+
if (!test) {
|
|
141
|
+
test = {
|
|
142
|
+
title: toFKTestTitle(attemptData.gherkinDocument, attemptData.pickle),
|
|
143
|
+
location: attemptData.pickle.location ? createLocation(worktree, this.cwd, featureUri, attemptData.pickle.location) : void 0,
|
|
144
|
+
tags: attemptData.pickle.tags.map((tag) => stripTagPrefix(tag.name)),
|
|
145
|
+
attempts: []
|
|
146
|
+
};
|
|
147
|
+
testsById.set(attemptData.testCase.id, test);
|
|
148
|
+
parentSuite.tests.push(test);
|
|
149
|
+
}
|
|
150
|
+
const testCaseFinished = this._testCaseFinishedById.get(testCaseStartedId);
|
|
151
|
+
const startTimestamp = toUnixTimestampMS(testCaseStarted.timestamp);
|
|
152
|
+
const finishTimestamp = testCaseFinished ? toUnixTimestampMS(testCaseFinished.timestamp) : startTimestamp;
|
|
153
|
+
const errors = parsedAttempt.testSteps.map((step) => extractErrorFromStep(worktree, this.cwd, step)).filter((error) => !!error);
|
|
154
|
+
const stdio = extractSTDIOFromTestSteps(parsedAttempt.testSteps, startTimestamp);
|
|
155
|
+
let parallelIndex;
|
|
156
|
+
if (testCaseStarted.workerId) {
|
|
157
|
+
parallelIndex = parallelIndexByWorkerId.get(testCaseStarted.workerId);
|
|
158
|
+
if (parallelIndex === void 0) {
|
|
159
|
+
parallelIndex = parallelIndexByWorkerId.size;
|
|
160
|
+
parallelIndexByWorkerId.set(testCaseStarted.workerId, parallelIndex);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
test.attempts.push({
|
|
164
|
+
environmentIdx: 0,
|
|
165
|
+
startTimestamp,
|
|
166
|
+
duration: Math.max(0, finishTimestamp - startTimestamp),
|
|
167
|
+
status: toFKStatus(attemptData.worstTestStepResult.status),
|
|
168
|
+
parallelIndex,
|
|
169
|
+
annotations: extractAttemptAnnotations(worktree, this.cwd, featureUri, attemptData.gherkinDocument, attemptData.pickle),
|
|
170
|
+
errors: errors.length ? errors : void 0,
|
|
171
|
+
attachments: await extractAttachmentsFromTestSteps(parsedAttempt.testSteps, attachments),
|
|
172
|
+
stdio: stdio.length ? stdio : void 0,
|
|
173
|
+
steps: parsedAttempt.testSteps.map((step) => ({
|
|
174
|
+
title: toFKStepTitle(step),
|
|
175
|
+
duration: toDurationMS(step.result.duration),
|
|
176
|
+
error: extractErrorFromStep(worktree, this.cwd, step),
|
|
177
|
+
location: step.sourceLocation ? createLineAndUriLocation(worktree, this.cwd, step.sourceLocation) : step.actionLocation ? createLineAndUriLocation(worktree, this.cwd, step.actionLocation) : void 0
|
|
178
|
+
}))
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
attachments: Array.from(attachments.values()),
|
|
183
|
+
suites: Array.from(suitesByKey.values()).filter((suite) => suite.type === "file")
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function envBool(name) {
|
|
188
|
+
return ["1", "true"].includes(process.env[name]?.toLowerCase() ?? "");
|
|
189
|
+
}
|
|
190
|
+
function parseFormatterConfig(parsedArgvOptions) {
|
|
191
|
+
return {
|
|
192
|
+
disableUpload: typeof parsedArgvOptions.disableUpload === "boolean" ? parsedArgvOptions.disableUpload : void 0,
|
|
193
|
+
endpoint: typeof parsedArgvOptions.endpoint === "string" ? parsedArgvOptions.endpoint : void 0,
|
|
194
|
+
flakinessProject: typeof parsedArgvOptions.flakinessProject === "string" ? parsedArgvOptions.flakinessProject : void 0,
|
|
195
|
+
outputFolder: typeof parsedArgvOptions.outputFolder === "string" ? parsedArgvOptions.outputFolder : void 0,
|
|
196
|
+
token: typeof parsedArgvOptions.token === "string" ? parsedArgvOptions.token : void 0
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function createLocation(worktree, cwd, relativeFile, location) {
|
|
200
|
+
return {
|
|
201
|
+
file: worktree.gitPath(canonicalizeAbsolutePath(path.resolve(cwd, relativeFile))),
|
|
202
|
+
line: location.line,
|
|
203
|
+
column: location.column ?? 1
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
function stripTagPrefix(tag) {
|
|
207
|
+
return tag.startsWith("@") ? tag.slice(1) : tag;
|
|
208
|
+
}
|
|
209
|
+
function getOrCreateFileSuite(suitesByKey, worktree, cwd, featureUri) {
|
|
210
|
+
const key = `file:${featureUri}`;
|
|
211
|
+
let suite = suitesByKey.get(key);
|
|
212
|
+
if (!suite) {
|
|
213
|
+
suite = {
|
|
214
|
+
type: "file",
|
|
215
|
+
title: path.basename(featureUri),
|
|
216
|
+
location: createLocation(worktree, cwd, featureUri, { line: 0, column: 0 }),
|
|
217
|
+
suites: []
|
|
218
|
+
};
|
|
219
|
+
suitesByKey.set(key, suite);
|
|
220
|
+
}
|
|
221
|
+
return suite;
|
|
222
|
+
}
|
|
223
|
+
function getOrCreateFeatureSuite(suitesByKey, fileSuite, worktree, cwd, featureUri, gherkinDocument) {
|
|
224
|
+
const key = `feature:${featureUri}`;
|
|
225
|
+
let suite = suitesByKey.get(key);
|
|
226
|
+
if (!suite) {
|
|
227
|
+
suite = {
|
|
228
|
+
type: "suite",
|
|
229
|
+
title: gherkinDocument.feature?.name ?? "",
|
|
230
|
+
location: gherkinDocument.feature?.location ? createLocation(worktree, cwd, featureUri, gherkinDocument.feature.location) : void 0,
|
|
231
|
+
suites: [],
|
|
232
|
+
tests: []
|
|
233
|
+
};
|
|
234
|
+
suitesByKey.set(key, suite);
|
|
235
|
+
fileSuite.suites.push(suite);
|
|
236
|
+
}
|
|
237
|
+
return suite;
|
|
238
|
+
}
|
|
239
|
+
function getOrCreateRuleSuite(suitesByKey, featureSuite, worktree, cwd, featureUri, rule) {
|
|
240
|
+
const key = `rule:${featureUri}:${rule.id}`;
|
|
241
|
+
let suite = suitesByKey.get(key);
|
|
242
|
+
if (!suite) {
|
|
243
|
+
suite = {
|
|
244
|
+
type: "suite",
|
|
245
|
+
title: rule.name,
|
|
246
|
+
location: createLocation(worktree, cwd, featureUri, rule.location),
|
|
247
|
+
tests: []
|
|
248
|
+
};
|
|
249
|
+
suitesByKey.set(key, suite);
|
|
250
|
+
featureSuite.suites.push(suite);
|
|
251
|
+
}
|
|
252
|
+
return suite;
|
|
253
|
+
}
|
|
254
|
+
function extractAttemptAnnotations(worktree, cwd, featureUri, gherkinDocument, pickle) {
|
|
255
|
+
const annotations = [
|
|
256
|
+
createDescriptionAnnotation("feature", worktree, cwd, featureUri, gherkinDocument.feature),
|
|
257
|
+
createDescriptionAnnotation("rule", worktree, cwd, featureUri, findRuleForPickle(gherkinDocument, pickle)),
|
|
258
|
+
createDescriptionAnnotation("scenario", worktree, cwd, featureUri, findScenarioForPickle(gherkinDocument, pickle))
|
|
259
|
+
].filter((annotation) => !!annotation);
|
|
260
|
+
return annotations.length ? annotations : void 0;
|
|
261
|
+
}
|
|
262
|
+
function createDescriptionAnnotation(type, worktree, cwd, featureUri, node) {
|
|
263
|
+
const description = normalizeDescription(node?.description);
|
|
264
|
+
if (!description || !node)
|
|
265
|
+
return void 0;
|
|
266
|
+
return {
|
|
267
|
+
type,
|
|
268
|
+
description,
|
|
269
|
+
location: createLocation(worktree, cwd, featureUri, node.location)
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
function findRuleForPickle(gherkinDocument, pickle) {
|
|
273
|
+
const astNodeIds = new Set(pickle.astNodeIds);
|
|
274
|
+
for (const child of gherkinDocument.feature?.children ?? []) {
|
|
275
|
+
if (!child.rule)
|
|
276
|
+
continue;
|
|
277
|
+
const hasScenario = child.rule.children.some((ruleChild) => ruleChild.scenario && astNodeIds.has(ruleChild.scenario.id));
|
|
278
|
+
if (hasScenario)
|
|
279
|
+
return child.rule;
|
|
280
|
+
}
|
|
281
|
+
return void 0;
|
|
282
|
+
}
|
|
283
|
+
function findScenarioForPickle(gherkinDocument, pickle) {
|
|
284
|
+
const astNodeIds = new Set(pickle.astNodeIds);
|
|
285
|
+
return collectScenarios(gherkinDocument.feature).find((scenario) => astNodeIds.has(scenario.id));
|
|
286
|
+
}
|
|
287
|
+
function toFKTestTitle(gherkinDocument, pickle) {
|
|
288
|
+
const exampleValues = extractScenarioOutlineValues(gherkinDocument, pickle);
|
|
289
|
+
if (exampleValues)
|
|
290
|
+
return `${pickle.name} [${exampleValues.map(([key, value]) => `${key}=${value}`).join(", ")}]`;
|
|
291
|
+
return pickle.name;
|
|
292
|
+
}
|
|
293
|
+
function extractScenarioOutlineValues(gherkinDocument, pickle) {
|
|
294
|
+
if (pickle.astNodeIds.length < 2)
|
|
295
|
+
return void 0;
|
|
296
|
+
const exampleRowId = pickle.astNodeIds[pickle.astNodeIds.length - 1];
|
|
297
|
+
for (const scenario of collectScenarios(gherkinDocument.feature)) {
|
|
298
|
+
for (const examples of scenario.examples) {
|
|
299
|
+
const row = examples.tableBody.find((row2) => row2.id === exampleRowId);
|
|
300
|
+
if (!row)
|
|
301
|
+
continue;
|
|
302
|
+
const headers = examples.tableHeader?.cells.map((cell) => cell.value) ?? [];
|
|
303
|
+
return row.cells.map((cell, index) => [headers[index] ?? `column${index + 1}`, cell.value]);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return void 0;
|
|
307
|
+
}
|
|
308
|
+
function collectScenarios(feature) {
|
|
309
|
+
return (feature?.children ?? []).flatMap((child) => {
|
|
310
|
+
if (child.rule)
|
|
311
|
+
return child.rule.children.flatMap((ruleChild) => ruleChild.scenario ? [ruleChild.scenario] : []);
|
|
312
|
+
return child.scenario ? [child.scenario] : [];
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
function normalizeDescription(description) {
|
|
316
|
+
const value = description?.trim();
|
|
317
|
+
if (!value)
|
|
318
|
+
return void 0;
|
|
319
|
+
const lines = value.split("\n");
|
|
320
|
+
const commonIndent = lines.slice(1).filter((line) => line.trim()).reduce((indent, line) => Math.min(indent, line.match(/^ */)?.[0].length ?? 0), Number.POSITIVE_INFINITY);
|
|
321
|
+
if (!Number.isFinite(commonIndent) || commonIndent === 0)
|
|
322
|
+
return value;
|
|
323
|
+
return [
|
|
324
|
+
lines[0],
|
|
325
|
+
...lines.slice(1).map((line) => line.slice(commonIndent))
|
|
326
|
+
].join("\n");
|
|
327
|
+
}
|
|
328
|
+
function toFKStatus(status) {
|
|
329
|
+
switch (status) {
|
|
330
|
+
case TestStepResultStatus.PASSED:
|
|
331
|
+
return "passed";
|
|
332
|
+
case TestStepResultStatus.SKIPPED:
|
|
333
|
+
return "skipped";
|
|
334
|
+
case TestStepResultStatus.UNKNOWN:
|
|
335
|
+
return "interrupted";
|
|
336
|
+
case TestStepResultStatus.PENDING:
|
|
337
|
+
case TestStepResultStatus.UNDEFINED:
|
|
338
|
+
case TestStepResultStatus.AMBIGUOUS:
|
|
339
|
+
case TestStepResultStatus.FAILED:
|
|
340
|
+
return "failed";
|
|
341
|
+
default:
|
|
342
|
+
return "interrupted";
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
function toUnixTimestampMS(timestamp) {
|
|
346
|
+
return timestamp.seconds * 1e3 + Math.floor(timestamp.nanos / 1e6);
|
|
347
|
+
}
|
|
348
|
+
function toDurationMS(timestamp) {
|
|
349
|
+
return timestamp.seconds * 1e3 + Math.floor(timestamp.nanos / 1e6);
|
|
350
|
+
}
|
|
351
|
+
function createLineAndUriLocation(worktree, cwd, location) {
|
|
352
|
+
return {
|
|
353
|
+
file: worktree.gitPath(canonicalizeAbsolutePath(path.resolve(cwd, location.uri))),
|
|
354
|
+
line: location.line,
|
|
355
|
+
column: 1
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
function canonicalizeAbsolutePath(absolutePath) {
|
|
359
|
+
try {
|
|
360
|
+
return fs.realpathSync.native(absolutePath);
|
|
361
|
+
} catch {
|
|
362
|
+
return absolutePath;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
function toFKStepTitle(step) {
|
|
366
|
+
return step.text ? `${step.keyword}${step.text}`.trim() : step.name ? `${step.keyword} (${step.name})` : step.keyword;
|
|
367
|
+
}
|
|
368
|
+
function extractErrorFromStep(worktree, cwd, step) {
|
|
369
|
+
const status = step.result.status;
|
|
370
|
+
if (status === TestStepResultStatus.PASSED || status === TestStepResultStatus.SKIPPED || status === TestStepResultStatus.UNKNOWN) {
|
|
371
|
+
return void 0;
|
|
372
|
+
}
|
|
373
|
+
const message = step.result.exception?.message ?? step.result.message ?? (status === TestStepResultStatus.PENDING ? "Step is pending" : status === TestStepResultStatus.UNDEFINED ? "Undefined step" : void 0);
|
|
374
|
+
const location = step.sourceLocation ? createLineAndUriLocation(worktree, cwd, step.sourceLocation) : step.actionLocation ? createLineAndUriLocation(worktree, cwd, step.actionLocation) : void 0;
|
|
375
|
+
return message ? {
|
|
376
|
+
location,
|
|
377
|
+
message,
|
|
378
|
+
stack: step.result.exception?.stackTrace,
|
|
379
|
+
snippet: step.snippet
|
|
380
|
+
} : void 0;
|
|
381
|
+
}
|
|
382
|
+
function extractSTDIOFromTestSteps(steps, startTimestamp) {
|
|
383
|
+
const stdio = [];
|
|
384
|
+
let previousTimestamp = startTimestamp;
|
|
385
|
+
for (const step of steps) {
|
|
386
|
+
for (const attachment of step.attachments) {
|
|
387
|
+
if (attachment.mediaType !== CUCUMBER_LOG_MEDIA_TYPE)
|
|
388
|
+
continue;
|
|
389
|
+
const timestamp = attachment.timestamp ? toUnixTimestampMS(attachment.timestamp) : previousTimestamp;
|
|
390
|
+
stdio.push({
|
|
391
|
+
...attachment.contentEncoding === AttachmentContentEncoding.BASE64 ? {
|
|
392
|
+
buffer: attachment.body
|
|
393
|
+
} : {
|
|
394
|
+
text: attachment.body
|
|
395
|
+
},
|
|
396
|
+
dts: Math.max(0, timestamp - previousTimestamp)
|
|
397
|
+
});
|
|
398
|
+
previousTimestamp = timestamp;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return stdio;
|
|
402
|
+
}
|
|
403
|
+
async function extractAttachmentsFromTestSteps(steps, attachments) {
|
|
404
|
+
const fkAttachments = [];
|
|
405
|
+
for (const step of steps) {
|
|
406
|
+
for (const attachment of step.attachments) {
|
|
407
|
+
if (attachment.mediaType === CUCUMBER_LOG_MEDIA_TYPE)
|
|
408
|
+
continue;
|
|
409
|
+
const dataAttachment = await ReportUtils.createDataAttachment(
|
|
410
|
+
attachment.mediaType,
|
|
411
|
+
decodeAttachmentBody(attachment)
|
|
412
|
+
);
|
|
413
|
+
attachments.set(dataAttachment.id, dataAttachment);
|
|
414
|
+
fkAttachments.push({
|
|
415
|
+
id: dataAttachment.id,
|
|
416
|
+
name: attachment.fileName ?? `attachment-${fkAttachments.length + 1}`,
|
|
417
|
+
contentType: attachment.mediaType
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return fkAttachments;
|
|
422
|
+
}
|
|
423
|
+
function decodeAttachmentBody(attachment) {
|
|
424
|
+
if (attachment.contentEncoding === AttachmentContentEncoding.BASE64)
|
|
425
|
+
return Buffer.from(attachment.body, "base64");
|
|
426
|
+
return Buffer.from(attachment.body, "utf8");
|
|
427
|
+
}
|
|
428
|
+
class ManualPromise {
|
|
429
|
+
promise;
|
|
430
|
+
_resolve;
|
|
431
|
+
_reject;
|
|
432
|
+
constructor() {
|
|
433
|
+
this.promise = new Promise((resolve, reject) => {
|
|
434
|
+
this._resolve = resolve;
|
|
435
|
+
this._reject = reject;
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
resolve(e) {
|
|
439
|
+
this._resolve(e);
|
|
440
|
+
}
|
|
441
|
+
reject(e) {
|
|
442
|
+
this._reject(e);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
export {
|
|
446
|
+
FlakinessCucumberFormatter as default
|
|
447
|
+
};
|
|
448
|
+
//# sourceMappingURL=formatter.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flakiness/cucumberjs",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"repository": {
|
|
5
|
+
"type": "git",
|
|
6
|
+
"url": "git+https://github.com/flakiness/cucumberjs.git"
|
|
7
|
+
},
|
|
4
8
|
"description": "",
|
|
5
9
|
"exports": {
|
|
6
10
|
".": {
|
|
@@ -11,11 +15,6 @@
|
|
|
11
15
|
},
|
|
12
16
|
"main": "./lib/formatter.js",
|
|
13
17
|
"types": "./types/src/formatter.d.ts",
|
|
14
|
-
"scripts": {
|
|
15
|
-
"build": "kubik build.mts",
|
|
16
|
-
"test": "cucumber-js",
|
|
17
|
-
"coverage": "c8 --reporter=text --reporter=html --include=lib/formatter.js cucumber-js"
|
|
18
|
-
},
|
|
19
18
|
"keywords": [],
|
|
20
19
|
"author": "",
|
|
21
20
|
"license": "MIT",
|
|
@@ -27,11 +26,17 @@
|
|
|
27
26
|
"devDependencies": {
|
|
28
27
|
"@cucumber/cucumber": "^12.7.0",
|
|
29
28
|
"@cucumber/messages": "^32.2.0",
|
|
29
|
+
"@flakiness/cucumberjs": "1.0.0-alpha.1",
|
|
30
30
|
"@types/node": "^25.5.0",
|
|
31
31
|
"c8": "^11.0.0",
|
|
32
32
|
"esbuild": "^0.27.4",
|
|
33
33
|
"kubik": "^0.24.0",
|
|
34
34
|
"tsx": "^4.21.0",
|
|
35
35
|
"typescript": "^5.9.3"
|
|
36
|
+
},
|
|
37
|
+
"scripts": {
|
|
38
|
+
"build": "kubik build.mts",
|
|
39
|
+
"test": "cucumber-js",
|
|
40
|
+
"coverage": "c8 --reporter=text --reporter=html --include=lib/formatter.js cucumber-js"
|
|
36
41
|
}
|
|
37
|
-
}
|
|
42
|
+
}
|