@flakiness/cucumberjs 1.0.0-alpha.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/formatter.js +438 -0
- package/package.json +7 -7
- package/types/src/formatter.d.ts +23 -0
- package/types/src/formatter.d.ts.map +1 -0
- package/.github/workflows/flakiness-upload-fork-prs.yml +0 -30
- package/.github/workflows/publish-npm.yml +0 -41
- package/.github/workflows/tests.yml +0 -45
- package/CONTRIBUTING.md +0 -58
- package/agenda.md +0 -2
- package/build.mts +0 -34
- package/cucumber.mjs +0 -6
- package/features/attachments.feature +0 -32
- package/features/basic.feature +0 -27
- package/features/data_tables.feature +0 -45
- package/features/description.feature +0 -49
- package/features/errors.feature +0 -28
- package/features/hooks_named.feature +0 -32
- package/features/hooks_unnamed.feature +0 -33
- package/features/locations.feature +0 -37
- package/features/retries.feature +0 -30
- package/features/rules.feature +0 -25
- package/features/scenario_outlines.feature +0 -57
- package/features/scenario_outlines_multiple.feature +0 -44
- package/features/statuses.feature +0 -70
- package/features/stdio.feature +0 -29
- package/features/steps.feature +0 -24
- package/features/support/attachments_steps.ts +0 -32
- package/features/support/basic_steps.ts +0 -235
- package/features/support/description_steps.ts +0 -37
- package/features/support/errors_steps.ts +0 -48
- package/features/support/harness.ts +0 -196
- package/features/support/project_steps.ts +0 -24
- package/features/support/stdio_steps.ts +0 -21
- package/features/support/tags_steps.ts +0 -10
- package/features/tags.feature +0 -19
- package/features/tags_hierarchy.feature +0 -37
- package/plan.md +0 -59
- package/pnpm-workspace.yaml +0 -2
- package/src/formatter.ts +0 -635
- package/tsconfig.json +0 -24
package/lib/formatter.js
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
import { Formatter, formatterHelpers } from "@cucumber/cucumber";
|
|
2
|
+
import { AttachmentContentEncoding, TestStepResultStatus } from "@cucumber/messages";
|
|
3
|
+
import {
|
|
4
|
+
CIUtils,
|
|
5
|
+
CPUUtilization,
|
|
6
|
+
GitWorktree,
|
|
7
|
+
RAMUtilization,
|
|
8
|
+
ReportUtils,
|
|
9
|
+
uploadReport,
|
|
10
|
+
writeReport
|
|
11
|
+
} from "@flakiness/sdk";
|
|
12
|
+
import fs from "node:fs";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
const CUCUMBER_LOG_MEDIA_TYPE = "text/x.cucumber.log+plain";
|
|
15
|
+
class FlakinessCucumberFormatter extends Formatter {
|
|
16
|
+
static documentation = "Generates a Flakiness report for a CucumberJS run.";
|
|
17
|
+
_config;
|
|
18
|
+
_cpuUtilization = new CPUUtilization({ precision: 10 });
|
|
19
|
+
_ramUtilization = new RAMUtilization({ precision: 10 });
|
|
20
|
+
_startTimestamp = Date.now();
|
|
21
|
+
_outputFolder;
|
|
22
|
+
_telemetryTimer;
|
|
23
|
+
_finishedPromise = new ManualPromise();
|
|
24
|
+
_testCaseStartedById = /* @__PURE__ */ new Map();
|
|
25
|
+
_testCaseFinishedById = /* @__PURE__ */ new Map();
|
|
26
|
+
constructor(options) {
|
|
27
|
+
super(options);
|
|
28
|
+
this._config = parseFormatterConfig(options.parsedArgvOptions);
|
|
29
|
+
this._outputFolder = path.resolve(this.cwd, this._config.outputFolder ?? process.env.FLAKINESS_OUTPUT_DIR ?? "flakiness-report");
|
|
30
|
+
this._sampleSystem = this._sampleSystem.bind(this);
|
|
31
|
+
this._sampleSystem();
|
|
32
|
+
options.eventBroadcaster.on("envelope", (envelope) => {
|
|
33
|
+
if (envelope.testRunStarted)
|
|
34
|
+
this._onTestRunStarted(envelope.testRunStarted);
|
|
35
|
+
if (envelope.testCaseStarted)
|
|
36
|
+
this._onTestCaseStarted(envelope.testCaseStarted);
|
|
37
|
+
if (envelope.testCaseFinished)
|
|
38
|
+
this._onTestCaseFinished(envelope.testCaseFinished);
|
|
39
|
+
if (envelope.testRunFinished) {
|
|
40
|
+
this._onTestRunFinished(envelope.testRunFinished).then(() => this._finishedPromise.resolve(void 0)).catch((e) => this._finishedPromise.reject(e));
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
_onTestRunStarted(testRunStarted) {
|
|
45
|
+
this._startTimestamp = toUnixTimestampMS(testRunStarted.timestamp);
|
|
46
|
+
}
|
|
47
|
+
_onTestCaseStarted(testCaseStarted) {
|
|
48
|
+
this._testCaseStartedById.set(testCaseStarted.id, testCaseStarted);
|
|
49
|
+
}
|
|
50
|
+
_onTestCaseFinished(testCaseFinished) {
|
|
51
|
+
this._testCaseFinishedById.set(testCaseFinished.testCaseStartedId, testCaseFinished);
|
|
52
|
+
}
|
|
53
|
+
async finished() {
|
|
54
|
+
if (this._telemetryTimer)
|
|
55
|
+
clearTimeout(this._telemetryTimer);
|
|
56
|
+
try {
|
|
57
|
+
await this._finishedPromise.promise;
|
|
58
|
+
} catch (error) {
|
|
59
|
+
console.error(`[flakiness.io] Failed to generate report: ${error instanceof Error ? error.stack ?? error.message : String(error)}`);
|
|
60
|
+
}
|
|
61
|
+
await super.finished();
|
|
62
|
+
}
|
|
63
|
+
_sampleSystem() {
|
|
64
|
+
this._cpuUtilization.sample();
|
|
65
|
+
this._ramUtilization.sample();
|
|
66
|
+
this._telemetryTimer = setTimeout(this._sampleSystem, 1e3);
|
|
67
|
+
}
|
|
68
|
+
async _onTestRunFinished(testRunFinished) {
|
|
69
|
+
this._cpuUtilization.sample();
|
|
70
|
+
this._ramUtilization.sample();
|
|
71
|
+
let worktree;
|
|
72
|
+
let commitId;
|
|
73
|
+
try {
|
|
74
|
+
worktree = GitWorktree.create(this.cwd);
|
|
75
|
+
commitId = worktree.headCommitId();
|
|
76
|
+
} catch {
|
|
77
|
+
console.warn("[flakiness.io] Failed to fetch commit info - is this a git repo?");
|
|
78
|
+
console.error("[flakiness.io] Report is NOT generated.");
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const { attachments, suites } = await this._collectSuites(worktree);
|
|
82
|
+
const report = ReportUtils.normalizeReport({
|
|
83
|
+
category: "cucumberjs",
|
|
84
|
+
commitId,
|
|
85
|
+
duration: toUnixTimestampMS(testRunFinished.timestamp) - this._startTimestamp,
|
|
86
|
+
environments: [
|
|
87
|
+
ReportUtils.createEnvironment({
|
|
88
|
+
name: "cucumberjs"
|
|
89
|
+
})
|
|
90
|
+
],
|
|
91
|
+
flakinessProject: this._config.flakinessProject,
|
|
92
|
+
suites,
|
|
93
|
+
startTimestamp: this._startTimestamp,
|
|
94
|
+
url: CIUtils.runUrl()
|
|
95
|
+
});
|
|
96
|
+
ReportUtils.collectSources(worktree, report);
|
|
97
|
+
this._cpuUtilization.enrich(report);
|
|
98
|
+
this._ramUtilization.enrich(report);
|
|
99
|
+
await writeReport(report, attachments, this._outputFolder);
|
|
100
|
+
const disableUpload = this._config.disableUpload ?? envBool("FLAKINESS_DISABLE_UPLOAD");
|
|
101
|
+
if (!disableUpload) {
|
|
102
|
+
await uploadReport(report, attachments, {
|
|
103
|
+
flakinessAccessToken: this._config.token,
|
|
104
|
+
flakinessEndpoint: this._config.endpoint
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
const defaultOutputFolder = path.join(this.cwd, "flakiness-report");
|
|
108
|
+
const folder = defaultOutputFolder === this._outputFolder ? "" : path.relative(this.cwd, this._outputFolder);
|
|
109
|
+
this.log(`
|
|
110
|
+
To open last Flakiness report, run:
|
|
111
|
+
|
|
112
|
+
npx flakiness show ${folder}
|
|
113
|
+
`);
|
|
114
|
+
}
|
|
115
|
+
async _collectSuites(worktree) {
|
|
116
|
+
const suitesByKey = /* @__PURE__ */ new Map();
|
|
117
|
+
const testsById = /* @__PURE__ */ new Map();
|
|
118
|
+
const attachments = /* @__PURE__ */ new Map();
|
|
119
|
+
for (const [testCaseStartedId, testCaseStarted] of this._testCaseStartedById) {
|
|
120
|
+
const attemptData = this.eventDataCollector.getTestCaseAttempt(testCaseStartedId);
|
|
121
|
+
const parsedAttempt = formatterHelpers.parseTestCaseAttempt({
|
|
122
|
+
testCaseAttempt: attemptData,
|
|
123
|
+
snippetBuilder: this.snippetBuilder,
|
|
124
|
+
supportCodeLibrary: this.supportCodeLibrary
|
|
125
|
+
});
|
|
126
|
+
const featureUri = attemptData.pickle.uri;
|
|
127
|
+
const fileSuite = getOrCreateFileSuite(suitesByKey, worktree, this.cwd, featureUri);
|
|
128
|
+
const featureSuite = getOrCreateFeatureSuite(
|
|
129
|
+
suitesByKey,
|
|
130
|
+
fileSuite,
|
|
131
|
+
worktree,
|
|
132
|
+
this.cwd,
|
|
133
|
+
featureUri,
|
|
134
|
+
attemptData.gherkinDocument
|
|
135
|
+
);
|
|
136
|
+
const rule = findRuleForPickle(attemptData.gherkinDocument, attemptData.pickle);
|
|
137
|
+
const parentSuite = rule ? getOrCreateRuleSuite(suitesByKey, featureSuite, worktree, this.cwd, featureUri, rule) : featureSuite;
|
|
138
|
+
let test = testsById.get(attemptData.testCase.id);
|
|
139
|
+
if (!test) {
|
|
140
|
+
test = {
|
|
141
|
+
title: toFKTestTitle(attemptData.gherkinDocument, attemptData.pickle),
|
|
142
|
+
location: attemptData.pickle.location ? createLocation(worktree, this.cwd, featureUri, attemptData.pickle.location) : void 0,
|
|
143
|
+
tags: attemptData.pickle.tags.map((tag) => stripTagPrefix(tag.name)),
|
|
144
|
+
attempts: []
|
|
145
|
+
};
|
|
146
|
+
testsById.set(attemptData.testCase.id, test);
|
|
147
|
+
parentSuite.tests.push(test);
|
|
148
|
+
}
|
|
149
|
+
const testCaseFinished = this._testCaseFinishedById.get(testCaseStartedId);
|
|
150
|
+
const startTimestamp = toUnixTimestampMS(testCaseStarted.timestamp);
|
|
151
|
+
const finishTimestamp = testCaseFinished ? toUnixTimestampMS(testCaseFinished.timestamp) : startTimestamp;
|
|
152
|
+
const errors = parsedAttempt.testSteps.map((step) => extractErrorFromStep(worktree, this.cwd, step)).filter((error) => !!error);
|
|
153
|
+
const stdio = extractSTDIOFromTestSteps(parsedAttempt.testSteps, startTimestamp);
|
|
154
|
+
test.attempts.push({
|
|
155
|
+
environmentIdx: 0,
|
|
156
|
+
startTimestamp,
|
|
157
|
+
duration: Math.max(0, finishTimestamp - startTimestamp),
|
|
158
|
+
status: toFKStatus(attemptData.worstTestStepResult.status),
|
|
159
|
+
annotations: extractAttemptAnnotations(worktree, this.cwd, featureUri, attemptData.gherkinDocument, attemptData.pickle),
|
|
160
|
+
errors: errors.length ? errors : void 0,
|
|
161
|
+
attachments: await extractAttachmentsFromTestSteps(parsedAttempt.testSteps, attachments),
|
|
162
|
+
stdio: stdio.length ? stdio : void 0,
|
|
163
|
+
steps: parsedAttempt.testSteps.map((step) => ({
|
|
164
|
+
title: toFKStepTitle(step),
|
|
165
|
+
duration: toDurationMS(step.result.duration),
|
|
166
|
+
error: extractErrorFromStep(worktree, this.cwd, step),
|
|
167
|
+
location: step.sourceLocation ? createLineAndUriLocation(worktree, this.cwd, step.sourceLocation) : step.actionLocation ? createLineAndUriLocation(worktree, this.cwd, step.actionLocation) : void 0
|
|
168
|
+
}))
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
attachments: Array.from(attachments.values()),
|
|
173
|
+
suites: Array.from(suitesByKey.values()).filter((suite) => suite.type === "file")
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
function envBool(name) {
|
|
178
|
+
return ["1", "true"].includes(process.env[name]?.toLowerCase() ?? "");
|
|
179
|
+
}
|
|
180
|
+
function parseFormatterConfig(parsedArgvOptions) {
|
|
181
|
+
return {
|
|
182
|
+
disableUpload: typeof parsedArgvOptions.disableUpload === "boolean" ? parsedArgvOptions.disableUpload : void 0,
|
|
183
|
+
endpoint: typeof parsedArgvOptions.endpoint === "string" ? parsedArgvOptions.endpoint : void 0,
|
|
184
|
+
flakinessProject: typeof parsedArgvOptions.flakinessProject === "string" ? parsedArgvOptions.flakinessProject : void 0,
|
|
185
|
+
outputFolder: typeof parsedArgvOptions.outputFolder === "string" ? parsedArgvOptions.outputFolder : void 0,
|
|
186
|
+
token: typeof parsedArgvOptions.token === "string" ? parsedArgvOptions.token : void 0
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function createLocation(worktree, cwd, relativeFile, location) {
|
|
190
|
+
return {
|
|
191
|
+
file: worktree.gitPath(canonicalizeAbsolutePath(path.resolve(cwd, relativeFile))),
|
|
192
|
+
line: location.line,
|
|
193
|
+
column: location.column ?? 1
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
function stripTagPrefix(tag) {
|
|
197
|
+
return tag.startsWith("@") ? tag.slice(1) : tag;
|
|
198
|
+
}
|
|
199
|
+
function getOrCreateFileSuite(suitesByKey, worktree, cwd, featureUri) {
|
|
200
|
+
const key = `file:${featureUri}`;
|
|
201
|
+
let suite = suitesByKey.get(key);
|
|
202
|
+
if (!suite) {
|
|
203
|
+
suite = {
|
|
204
|
+
type: "file",
|
|
205
|
+
title: path.basename(featureUri),
|
|
206
|
+
location: createLocation(worktree, cwd, featureUri, { line: 0, column: 0 }),
|
|
207
|
+
suites: []
|
|
208
|
+
};
|
|
209
|
+
suitesByKey.set(key, suite);
|
|
210
|
+
}
|
|
211
|
+
return suite;
|
|
212
|
+
}
|
|
213
|
+
function getOrCreateFeatureSuite(suitesByKey, fileSuite, worktree, cwd, featureUri, gherkinDocument) {
|
|
214
|
+
const key = `feature:${featureUri}`;
|
|
215
|
+
let suite = suitesByKey.get(key);
|
|
216
|
+
if (!suite) {
|
|
217
|
+
suite = {
|
|
218
|
+
type: "suite",
|
|
219
|
+
title: gherkinDocument.feature?.name ?? "",
|
|
220
|
+
location: gherkinDocument.feature?.location ? createLocation(worktree, cwd, featureUri, gherkinDocument.feature.location) : void 0,
|
|
221
|
+
suites: [],
|
|
222
|
+
tests: []
|
|
223
|
+
};
|
|
224
|
+
suitesByKey.set(key, suite);
|
|
225
|
+
fileSuite.suites.push(suite);
|
|
226
|
+
}
|
|
227
|
+
return suite;
|
|
228
|
+
}
|
|
229
|
+
function getOrCreateRuleSuite(suitesByKey, featureSuite, worktree, cwd, featureUri, rule) {
|
|
230
|
+
const key = `rule:${featureUri}:${rule.id}`;
|
|
231
|
+
let suite = suitesByKey.get(key);
|
|
232
|
+
if (!suite) {
|
|
233
|
+
suite = {
|
|
234
|
+
type: "suite",
|
|
235
|
+
title: rule.name,
|
|
236
|
+
location: createLocation(worktree, cwd, featureUri, rule.location),
|
|
237
|
+
tests: []
|
|
238
|
+
};
|
|
239
|
+
suitesByKey.set(key, suite);
|
|
240
|
+
featureSuite.suites.push(suite);
|
|
241
|
+
}
|
|
242
|
+
return suite;
|
|
243
|
+
}
|
|
244
|
+
function extractAttemptAnnotations(worktree, cwd, featureUri, gherkinDocument, pickle) {
|
|
245
|
+
const annotations = [
|
|
246
|
+
createDescriptionAnnotation("feature", worktree, cwd, featureUri, gherkinDocument.feature),
|
|
247
|
+
createDescriptionAnnotation("rule", worktree, cwd, featureUri, findRuleForPickle(gherkinDocument, pickle)),
|
|
248
|
+
createDescriptionAnnotation("scenario", worktree, cwd, featureUri, findScenarioForPickle(gherkinDocument, pickle))
|
|
249
|
+
].filter((annotation) => !!annotation);
|
|
250
|
+
return annotations.length ? annotations : void 0;
|
|
251
|
+
}
|
|
252
|
+
function createDescriptionAnnotation(type, worktree, cwd, featureUri, node) {
|
|
253
|
+
const description = normalizeDescription(node?.description);
|
|
254
|
+
if (!description || !node)
|
|
255
|
+
return void 0;
|
|
256
|
+
return {
|
|
257
|
+
type,
|
|
258
|
+
description,
|
|
259
|
+
location: createLocation(worktree, cwd, featureUri, node.location)
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
function findRuleForPickle(gherkinDocument, pickle) {
|
|
263
|
+
const astNodeIds = new Set(pickle.astNodeIds);
|
|
264
|
+
for (const child of gherkinDocument.feature?.children ?? []) {
|
|
265
|
+
if (!child.rule)
|
|
266
|
+
continue;
|
|
267
|
+
const hasScenario = child.rule.children.some((ruleChild) => ruleChild.scenario && astNodeIds.has(ruleChild.scenario.id));
|
|
268
|
+
if (hasScenario)
|
|
269
|
+
return child.rule;
|
|
270
|
+
}
|
|
271
|
+
return void 0;
|
|
272
|
+
}
|
|
273
|
+
function findScenarioForPickle(gherkinDocument, pickle) {
|
|
274
|
+
const astNodeIds = new Set(pickle.astNodeIds);
|
|
275
|
+
return collectScenarios(gherkinDocument.feature).find((scenario) => astNodeIds.has(scenario.id));
|
|
276
|
+
}
|
|
277
|
+
function toFKTestTitle(gherkinDocument, pickle) {
|
|
278
|
+
const exampleValues = extractScenarioOutlineValues(gherkinDocument, pickle);
|
|
279
|
+
if (exampleValues)
|
|
280
|
+
return `${pickle.name} [${exampleValues.map(([key, value]) => `${key}=${value}`).join(", ")}]`;
|
|
281
|
+
return pickle.name;
|
|
282
|
+
}
|
|
283
|
+
function extractScenarioOutlineValues(gherkinDocument, pickle) {
|
|
284
|
+
if (pickle.astNodeIds.length < 2)
|
|
285
|
+
return void 0;
|
|
286
|
+
const exampleRowId = pickle.astNodeIds[pickle.astNodeIds.length - 1];
|
|
287
|
+
for (const scenario of collectScenarios(gherkinDocument.feature)) {
|
|
288
|
+
for (const examples of scenario.examples) {
|
|
289
|
+
const row = examples.tableBody.find((row2) => row2.id === exampleRowId);
|
|
290
|
+
if (!row)
|
|
291
|
+
continue;
|
|
292
|
+
const headers = examples.tableHeader?.cells.map((cell) => cell.value) ?? [];
|
|
293
|
+
return row.cells.map((cell, index) => [headers[index] ?? `column${index + 1}`, cell.value]);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return void 0;
|
|
297
|
+
}
|
|
298
|
+
function collectScenarios(feature) {
|
|
299
|
+
return (feature?.children ?? []).flatMap((child) => {
|
|
300
|
+
if (child.rule)
|
|
301
|
+
return child.rule.children.flatMap((ruleChild) => ruleChild.scenario ? [ruleChild.scenario] : []);
|
|
302
|
+
return child.scenario ? [child.scenario] : [];
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
function normalizeDescription(description) {
|
|
306
|
+
const value = description?.trim();
|
|
307
|
+
if (!value)
|
|
308
|
+
return void 0;
|
|
309
|
+
const lines = value.split("\n");
|
|
310
|
+
const commonIndent = lines.slice(1).filter((line) => line.trim()).reduce((indent, line) => Math.min(indent, line.match(/^ */)?.[0].length ?? 0), Number.POSITIVE_INFINITY);
|
|
311
|
+
if (!Number.isFinite(commonIndent) || commonIndent === 0)
|
|
312
|
+
return value;
|
|
313
|
+
return [
|
|
314
|
+
lines[0],
|
|
315
|
+
...lines.slice(1).map((line) => line.slice(commonIndent))
|
|
316
|
+
].join("\n");
|
|
317
|
+
}
|
|
318
|
+
function toFKStatus(status) {
|
|
319
|
+
switch (status) {
|
|
320
|
+
case TestStepResultStatus.PASSED:
|
|
321
|
+
return "passed";
|
|
322
|
+
case TestStepResultStatus.SKIPPED:
|
|
323
|
+
return "skipped";
|
|
324
|
+
case TestStepResultStatus.UNKNOWN:
|
|
325
|
+
return "interrupted";
|
|
326
|
+
case TestStepResultStatus.PENDING:
|
|
327
|
+
case TestStepResultStatus.UNDEFINED:
|
|
328
|
+
case TestStepResultStatus.AMBIGUOUS:
|
|
329
|
+
case TestStepResultStatus.FAILED:
|
|
330
|
+
return "failed";
|
|
331
|
+
default:
|
|
332
|
+
return "interrupted";
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
function toUnixTimestampMS(timestamp) {
|
|
336
|
+
return timestamp.seconds * 1e3 + Math.floor(timestamp.nanos / 1e6);
|
|
337
|
+
}
|
|
338
|
+
function toDurationMS(timestamp) {
|
|
339
|
+
return timestamp.seconds * 1e3 + Math.floor(timestamp.nanos / 1e6);
|
|
340
|
+
}
|
|
341
|
+
function createLineAndUriLocation(worktree, cwd, location) {
|
|
342
|
+
return {
|
|
343
|
+
file: worktree.gitPath(canonicalizeAbsolutePath(path.resolve(cwd, location.uri))),
|
|
344
|
+
line: location.line,
|
|
345
|
+
column: 1
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
function canonicalizeAbsolutePath(absolutePath) {
|
|
349
|
+
try {
|
|
350
|
+
return fs.realpathSync.native(absolutePath);
|
|
351
|
+
} catch {
|
|
352
|
+
return absolutePath;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
function toFKStepTitle(step) {
|
|
356
|
+
return step.text ? `${step.keyword}${step.text}`.trim() : step.name ? `${step.keyword} (${step.name})` : step.keyword;
|
|
357
|
+
}
|
|
358
|
+
function extractErrorFromStep(worktree, cwd, step) {
|
|
359
|
+
const status = step.result.status;
|
|
360
|
+
if (status === TestStepResultStatus.PASSED || status === TestStepResultStatus.SKIPPED || status === TestStepResultStatus.UNKNOWN) {
|
|
361
|
+
return void 0;
|
|
362
|
+
}
|
|
363
|
+
const message = step.result.exception?.message ?? step.result.message ?? (status === TestStepResultStatus.PENDING ? "Step is pending" : status === TestStepResultStatus.UNDEFINED ? "Undefined step" : void 0);
|
|
364
|
+
const location = step.sourceLocation ? createLineAndUriLocation(worktree, cwd, step.sourceLocation) : step.actionLocation ? createLineAndUriLocation(worktree, cwd, step.actionLocation) : void 0;
|
|
365
|
+
return message ? {
|
|
366
|
+
location,
|
|
367
|
+
message,
|
|
368
|
+
stack: step.result.exception?.stackTrace,
|
|
369
|
+
snippet: step.snippet
|
|
370
|
+
} : void 0;
|
|
371
|
+
}
|
|
372
|
+
function extractSTDIOFromTestSteps(steps, startTimestamp) {
|
|
373
|
+
const stdio = [];
|
|
374
|
+
let previousTimestamp = startTimestamp;
|
|
375
|
+
for (const step of steps) {
|
|
376
|
+
for (const attachment of step.attachments) {
|
|
377
|
+
if (attachment.mediaType !== CUCUMBER_LOG_MEDIA_TYPE)
|
|
378
|
+
continue;
|
|
379
|
+
const timestamp = attachment.timestamp ? toUnixTimestampMS(attachment.timestamp) : previousTimestamp;
|
|
380
|
+
stdio.push({
|
|
381
|
+
...attachment.contentEncoding === AttachmentContentEncoding.BASE64 ? {
|
|
382
|
+
buffer: attachment.body
|
|
383
|
+
} : {
|
|
384
|
+
text: attachment.body
|
|
385
|
+
},
|
|
386
|
+
dts: Math.max(0, timestamp - previousTimestamp)
|
|
387
|
+
});
|
|
388
|
+
previousTimestamp = timestamp;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return stdio;
|
|
392
|
+
}
|
|
393
|
+
async function extractAttachmentsFromTestSteps(steps, attachments) {
|
|
394
|
+
const fkAttachments = [];
|
|
395
|
+
for (const step of steps) {
|
|
396
|
+
for (const attachment of step.attachments) {
|
|
397
|
+
if (attachment.mediaType === CUCUMBER_LOG_MEDIA_TYPE)
|
|
398
|
+
continue;
|
|
399
|
+
const dataAttachment = await ReportUtils.createDataAttachment(
|
|
400
|
+
attachment.mediaType,
|
|
401
|
+
decodeAttachmentBody(attachment)
|
|
402
|
+
);
|
|
403
|
+
attachments.set(dataAttachment.id, dataAttachment);
|
|
404
|
+
fkAttachments.push({
|
|
405
|
+
id: dataAttachment.id,
|
|
406
|
+
name: attachment.fileName ?? `attachment-${fkAttachments.length + 1}`,
|
|
407
|
+
contentType: attachment.mediaType
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return fkAttachments;
|
|
412
|
+
}
|
|
413
|
+
function decodeAttachmentBody(attachment) {
|
|
414
|
+
if (attachment.contentEncoding === AttachmentContentEncoding.BASE64)
|
|
415
|
+
return Buffer.from(attachment.body, "base64");
|
|
416
|
+
return Buffer.from(attachment.body, "utf8");
|
|
417
|
+
}
|
|
418
|
+
class ManualPromise {
|
|
419
|
+
promise;
|
|
420
|
+
_resolve;
|
|
421
|
+
_reject;
|
|
422
|
+
constructor() {
|
|
423
|
+
this.promise = new Promise((resolve, reject) => {
|
|
424
|
+
this._resolve = resolve;
|
|
425
|
+
this._reject = reject;
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
resolve(e) {
|
|
429
|
+
this._resolve(e);
|
|
430
|
+
}
|
|
431
|
+
reject(e) {
|
|
432
|
+
this._reject(e);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
export {
|
|
436
|
+
FlakinessCucumberFormatter as default
|
|
437
|
+
};
|
|
438
|
+
//# sourceMappingURL=formatter.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flakiness/cucumberjs",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -11,11 +11,6 @@
|
|
|
11
11
|
},
|
|
12
12
|
"main": "./lib/formatter.js",
|
|
13
13
|
"types": "./types/src/formatter.d.ts",
|
|
14
|
-
"scripts": {
|
|
15
|
-
"build": "kubik build.mts",
|
|
16
|
-
"test": "cucumber-js",
|
|
17
|
-
"coverage": "c8 --reporter=text --reporter=html --include=lib/formatter.js cucumber-js"
|
|
18
|
-
},
|
|
19
14
|
"keywords": [],
|
|
20
15
|
"author": "",
|
|
21
16
|
"license": "MIT",
|
|
@@ -33,5 +28,10 @@
|
|
|
33
28
|
"kubik": "^0.24.0",
|
|
34
29
|
"tsx": "^4.21.0",
|
|
35
30
|
"typescript": "^5.9.3"
|
|
31
|
+
},
|
|
32
|
+
"scripts": {
|
|
33
|
+
"build": "kubik build.mts",
|
|
34
|
+
"test": "cucumber-js",
|
|
35
|
+
"coverage": "c8 --reporter=text --reporter=html --include=lib/formatter.js cucumber-js"
|
|
36
36
|
}
|
|
37
|
-
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { IFormatterOptions } from '@cucumber/cucumber';
|
|
2
|
+
import { Formatter } from '@cucumber/cucumber';
|
|
3
|
+
export default class FlakinessCucumberFormatter extends Formatter {
|
|
4
|
+
static documentation: string;
|
|
5
|
+
private _config;
|
|
6
|
+
private _cpuUtilization;
|
|
7
|
+
private _ramUtilization;
|
|
8
|
+
private _startTimestamp;
|
|
9
|
+
private _outputFolder;
|
|
10
|
+
private _telemetryTimer?;
|
|
11
|
+
private _finishedPromise;
|
|
12
|
+
private _testCaseStartedById;
|
|
13
|
+
private _testCaseFinishedById;
|
|
14
|
+
constructor(options: IFormatterOptions);
|
|
15
|
+
private _onTestRunStarted;
|
|
16
|
+
private _onTestCaseStarted;
|
|
17
|
+
private _onTestCaseFinished;
|
|
18
|
+
finished(): Promise<void>;
|
|
19
|
+
private _sampleSystem;
|
|
20
|
+
private _onTestRunFinished;
|
|
21
|
+
private _collectSuites;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=formatter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formatter.d.ts","sourceRoot":"","sources":["../../src/formatter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAoB,MAAM,oBAAoB,CAAC;AAiDjE,MAAM,CAAC,OAAO,OAAO,0BAA2B,SAAQ,SAAS;IAC/D,MAAM,CAAC,aAAa,SAAwD;IAE5E,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,eAAe,CAAyC;IAChE,OAAO,CAAC,eAAe,CAAyC;IAChE,OAAO,CAAC,eAAe,CAAoC;IAC3D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAiB;IAEzC,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,oBAAoB,CAAsC;IAClE,OAAO,CAAC,qBAAqB,CAAuC;gBAExD,OAAO,EAAE,iBAAiB;IA0BtC,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,mBAAmB;IAIZ,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAaxC,OAAO,CAAC,aAAa;YAMP,kBAAkB;YAsDlB,cAAc;CA+E7B"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
name: Upload Flakiness.io report (fork PRs)
|
|
2
|
-
on:
|
|
3
|
-
workflow_run:
|
|
4
|
-
# Must match the name(s) of workflows that produce flakiness-report artifacts
|
|
5
|
-
workflows: ["Tests"]
|
|
6
|
-
types: [completed]
|
|
7
|
-
|
|
8
|
-
jobs:
|
|
9
|
-
upload-flakiness-report:
|
|
10
|
-
runs-on: ubuntu-latest
|
|
11
|
-
if: >-
|
|
12
|
-
(github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')
|
|
13
|
-
&& github.event.workflow_run.event == 'pull_request'
|
|
14
|
-
&& github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name
|
|
15
|
-
permissions:
|
|
16
|
-
actions: read
|
|
17
|
-
contents: read
|
|
18
|
-
id-token: write
|
|
19
|
-
steps:
|
|
20
|
-
- name: Install Flakiness CLI
|
|
21
|
-
run: curl -LsSf https://cli.flakiness.io/install.sh | sh
|
|
22
|
-
|
|
23
|
-
- name: Download flakiness-report artifacts
|
|
24
|
-
env:
|
|
25
|
-
GH_TOKEN: ${{ github.token }}
|
|
26
|
-
RUN_ID: ${{ github.event.workflow_run.id }}
|
|
27
|
-
run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --pattern 'flakiness-report-*' --dir .
|
|
28
|
-
|
|
29
|
-
- name: Upload to Flakiness.io
|
|
30
|
-
run: find . -path '*/flakiness-report-*/report.json' -exec flakiness upload {} \;
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
name: Publish to NPM
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
release:
|
|
5
|
-
types: [published]
|
|
6
|
-
|
|
7
|
-
permissions:
|
|
8
|
-
id-token: write # Required for OIDC
|
|
9
|
-
contents: read
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
publish-to-npm:
|
|
13
|
-
runs-on: ubuntu-latest
|
|
14
|
-
|
|
15
|
-
steps:
|
|
16
|
-
- uses: actions/checkout@v4
|
|
17
|
-
|
|
18
|
-
- name: Setup Node.js
|
|
19
|
-
uses: actions/setup-node@v4
|
|
20
|
-
with:
|
|
21
|
-
node-version: 24
|
|
22
|
-
|
|
23
|
-
- name: Setup PNPM
|
|
24
|
-
uses: pnpm/action-setup@v4
|
|
25
|
-
with:
|
|
26
|
-
version: 10
|
|
27
|
-
|
|
28
|
-
- name: Build & Publish
|
|
29
|
-
run: |
|
|
30
|
-
pnpm i --frozen-lockfile
|
|
31
|
-
pnpm build
|
|
32
|
-
|
|
33
|
-
VERSION=${GITHUB_REF_NAME#v}
|
|
34
|
-
|
|
35
|
-
if [[ "$VERSION" == *"-"* ]]; then
|
|
36
|
-
echo "Publishing prerelease to @next"
|
|
37
|
-
pnpm publish --access=public --no-git-checks --tag next
|
|
38
|
-
else
|
|
39
|
-
echo "Publishing stable to @latest"
|
|
40
|
-
pnpm publish --access=public --no-git-checks
|
|
41
|
-
fi
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
name: Tests
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [main]
|
|
6
|
-
pull_request:
|
|
7
|
-
branches: [main]
|
|
8
|
-
|
|
9
|
-
permissions:
|
|
10
|
-
contents: read
|
|
11
|
-
id-token: write
|
|
12
|
-
|
|
13
|
-
jobs:
|
|
14
|
-
test:
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
19
|
-
runs-on: ${{ matrix.os }}
|
|
20
|
-
steps:
|
|
21
|
-
- uses: actions/checkout@v4
|
|
22
|
-
|
|
23
|
-
- name: Setup pnpm
|
|
24
|
-
uses: pnpm/action-setup@v4
|
|
25
|
-
with:
|
|
26
|
-
version: 10
|
|
27
|
-
|
|
28
|
-
- uses: actions/setup-node@v4
|
|
29
|
-
with:
|
|
30
|
-
node-version: lts/*
|
|
31
|
-
cache: pnpm
|
|
32
|
-
|
|
33
|
-
# Since tests initialize git repositories, we have
|
|
34
|
-
# to configure git to avoid warnings.
|
|
35
|
-
- run: git config --global init.defaultBranch main
|
|
36
|
-
- run: pnpm install --frozen-lockfile
|
|
37
|
-
- run: pnpm build
|
|
38
|
-
- run: pnpm test
|
|
39
|
-
- name: Upload Flakiness report artifact (for Pull Requests from forks)
|
|
40
|
-
if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
|
|
41
|
-
uses: actions/upload-artifact@v4
|
|
42
|
-
with:
|
|
43
|
-
name: flakiness-report-${{ github.job }}-${{ strategy.job-index }}
|
|
44
|
-
path: flakiness-report/
|
|
45
|
-
retention-days: 1
|
package/CONTRIBUTING.md
DELETED
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# Contributing
|
|
2
|
-
|
|
3
|
-
## Prerequisites
|
|
4
|
-
|
|
5
|
-
- Node.js 22+
|
|
6
|
-
- [pnpm](https://pnpm.io/)
|
|
7
|
-
|
|
8
|
-
## Getting Started
|
|
9
|
-
|
|
10
|
-
Clone the repo and install dependencies:
|
|
11
|
-
|
|
12
|
-
```bash
|
|
13
|
-
git clone https://github.com/flakiness/cucumberjs.git fk-cucumber
|
|
14
|
-
cd fk-cucumber
|
|
15
|
-
pnpm install
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
## Building
|
|
19
|
-
|
|
20
|
-
This project uses [Kubik](https://github.com/flakiness/kubik) as its build system. The build script is defined in `build.mts`.
|
|
21
|
-
|
|
22
|
-
To build:
|
|
23
|
-
|
|
24
|
-
```bash
|
|
25
|
-
pnpm build
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
To watch:
|
|
29
|
-
|
|
30
|
-
```bash
|
|
31
|
-
pnpm build -w
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
This will bundle the source with esbuild and generate TypeScript declarations.
|
|
35
|
-
|
|
36
|
-
## Releasing
|
|
37
|
-
|
|
38
|
-
To release a new version:
|
|
39
|
-
|
|
40
|
-
1. Bump the version:
|
|
41
|
-
|
|
42
|
-
```bash
|
|
43
|
-
# For a stable minor release
|
|
44
|
-
pnpm version minor
|
|
45
|
-
|
|
46
|
-
# For an alpha pre-release
|
|
47
|
-
pnpm version preminor --preid=alpha
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
2. Push the commit and tag:
|
|
51
|
-
|
|
52
|
-
```bash
|
|
53
|
-
git push --follow-tags
|
|
54
|
-
```
|
|
55
|
-
|
|
56
|
-
3. [Create a GitHub Release](https://github.com/flakiness/cucumberjs/releases/new) for the new tag and publish it.
|
|
57
|
-
|
|
58
|
-
CI will handle publishing to npm. Pre-releases are published under @next tag.
|
package/agenda.md
DELETED