@flakiness/report 0.92.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -35
- package/package.json +5 -44
- package/types/tsconfig.tsbuildinfo +1 -1
- package/lib/cli/flakiness.js +0 -168
- package/lib/createTestStepSnippets.js +0 -31
- package/lib/junit.js +0 -196
- package/lib/playwright-test.js +0 -235
- package/lib/playwrightJSONReport.js +0 -178
- package/lib/reportUploader.js +0 -115
- package/lib/systemUtilizationSampler.js +0 -70
- package/lib/utils.js +0 -322
- /package/lib/{common/flakinessReport.js → flakinessReport.js} +0 -0
- /package/lib/{common/schema.js → schema.js} +0 -0
package/lib/junit.js
DELETED
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
import { parseXml, XmlElement, XmlText } from "@rgrove/parse-xml";
|
|
2
|
-
import assert from "assert";
|
|
3
|
-
import fs from "fs";
|
|
4
|
-
import path from "path";
|
|
5
|
-
import { FlakinessReport as FK } from "./common/flakinessReport.js";
|
|
6
|
-
import { sha1Buffer, sha1File } from "./utils.js";
|
|
7
|
-
function getProperties(element) {
|
|
8
|
-
const propertiesNodes = element.children.filter((node) => node instanceof XmlElement).filter((node) => node.name === "properties");
|
|
9
|
-
if (!propertiesNodes.length)
|
|
10
|
-
return [];
|
|
11
|
-
const result = [];
|
|
12
|
-
for (const propertiesNode of propertiesNodes) {
|
|
13
|
-
const properties = propertiesNode.children.filter((node) => node instanceof XmlElement).filter((node) => node.name === "property");
|
|
14
|
-
for (const property of properties) {
|
|
15
|
-
const name = property.attributes["name"];
|
|
16
|
-
const innerText = property.children.find((node) => node instanceof XmlText);
|
|
17
|
-
const value = property.attributes["value"] ?? innerText?.text ?? "";
|
|
18
|
-
result.push([name, value]);
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
return result;
|
|
22
|
-
}
|
|
23
|
-
function extractErrors(testcase) {
|
|
24
|
-
const xmlErrors = testcase.children.filter((e) => e instanceof XmlElement).filter((element) => element.name === "error" || element.name === "failure");
|
|
25
|
-
if (!xmlErrors.length)
|
|
26
|
-
return void 0;
|
|
27
|
-
const errors = [];
|
|
28
|
-
for (const xmlErr of xmlErrors) {
|
|
29
|
-
const message = [xmlErr.attributes["type"], xmlErr.attributes["message"]].filter((x) => !!x).join(" ");
|
|
30
|
-
const xmlStackNodes = xmlErr.children.filter((child) => child instanceof XmlText);
|
|
31
|
-
const stack = xmlStackNodes ? xmlStackNodes.map((node) => node.text).join("\n") : void 0;
|
|
32
|
-
errors.push({
|
|
33
|
-
message,
|
|
34
|
-
stack
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
return errors;
|
|
38
|
-
}
|
|
39
|
-
function extractStdout(testcase, stdio) {
|
|
40
|
-
const xmlStdio = testcase.children.filter((e) => e instanceof XmlElement).filter((element) => element.name === stdio);
|
|
41
|
-
if (!xmlStdio.length)
|
|
42
|
-
return void 0;
|
|
43
|
-
return xmlStdio.map((node) => node.children.filter((node2) => node2 instanceof XmlText)).flat().map((txtNode) => ({
|
|
44
|
-
text: txtNode.text
|
|
45
|
-
}));
|
|
46
|
-
}
|
|
47
|
-
async function parseAttachment(value) {
|
|
48
|
-
let absolutePath = path.resolve(process.cwd(), value);
|
|
49
|
-
if (fs.existsSync(absolutePath)) {
|
|
50
|
-
const id = await sha1File(absolutePath);
|
|
51
|
-
return {
|
|
52
|
-
contentType: "image/png",
|
|
53
|
-
path: absolutePath,
|
|
54
|
-
id
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
contentType: "text/plain",
|
|
59
|
-
id: sha1Buffer(value),
|
|
60
|
-
body: Buffer.from(value)
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
async function traverseJUnitReport(context, node) {
|
|
64
|
-
const element = node;
|
|
65
|
-
if (!(element instanceof XmlElement))
|
|
66
|
-
return;
|
|
67
|
-
let { currentEnv, currentEnvIndex, currentSuite, report, currentTimeMs, attachments } = context;
|
|
68
|
-
if (element.attributes["timestamp"])
|
|
69
|
-
currentTimeMs = new Date(element.attributes["timestamp"]).getTime();
|
|
70
|
-
if (element.name === "testsuite") {
|
|
71
|
-
const file = element.attributes["file"];
|
|
72
|
-
const line = parseInt(element.attributes["line"], 10);
|
|
73
|
-
const name = element.attributes["name"];
|
|
74
|
-
const newSuite = {
|
|
75
|
-
title: name ?? file,
|
|
76
|
-
location: file && !isNaN(line) ? {
|
|
77
|
-
file,
|
|
78
|
-
line,
|
|
79
|
-
column: 1
|
|
80
|
-
} : FK.NO_LOCATION,
|
|
81
|
-
type: name ? "suite" : file ? "file" : "anonymous suite",
|
|
82
|
-
suites: [],
|
|
83
|
-
tests: []
|
|
84
|
-
};
|
|
85
|
-
if (currentSuite) {
|
|
86
|
-
currentSuite.suites ??= [];
|
|
87
|
-
currentSuite.suites.push(newSuite);
|
|
88
|
-
} else {
|
|
89
|
-
report.suites.push(newSuite);
|
|
90
|
-
}
|
|
91
|
-
currentSuite = newSuite;
|
|
92
|
-
const userSuppliedData = getProperties(element);
|
|
93
|
-
if (userSuppliedData.length) {
|
|
94
|
-
currentEnv = structuredClone(currentEnv);
|
|
95
|
-
currentEnv.userSuppliedData ??= {};
|
|
96
|
-
for (const [key, value] of userSuppliedData)
|
|
97
|
-
currentEnv.userSuppliedData[key] = value;
|
|
98
|
-
currentEnvIndex = report.environments.push(currentEnv) - 1;
|
|
99
|
-
}
|
|
100
|
-
} else if (element.name === "testcase") {
|
|
101
|
-
assert(currentSuite);
|
|
102
|
-
const file = element.attributes["file"];
|
|
103
|
-
const name = element.attributes["name"];
|
|
104
|
-
const line = parseInt(element.attributes["line"], 10);
|
|
105
|
-
const timeMs = parseFloat(element.attributes["time"]) * 1e3;
|
|
106
|
-
const startTimestamp = currentTimeMs;
|
|
107
|
-
const duration = timeMs;
|
|
108
|
-
currentTimeMs += timeMs;
|
|
109
|
-
const annotations = [];
|
|
110
|
-
const attachments2 = [];
|
|
111
|
-
for (const [key, value] of getProperties(element)) {
|
|
112
|
-
if (key.toLowerCase().startsWith("attachment")) {
|
|
113
|
-
if (context.ignoreAttachments)
|
|
114
|
-
continue;
|
|
115
|
-
const attachment = await parseAttachment(value);
|
|
116
|
-
context.attachments.set(attachment.id, attachment);
|
|
117
|
-
attachments2.push({
|
|
118
|
-
id: attachment.id,
|
|
119
|
-
contentType: attachment.contentType,
|
|
120
|
-
//TODO: better default names for attachments?
|
|
121
|
-
name: attachment.path ? path.basename(attachment.path) : `attachment`
|
|
122
|
-
});
|
|
123
|
-
} else {
|
|
124
|
-
annotations.push({
|
|
125
|
-
type: key,
|
|
126
|
-
description: value.length ? value : void 0
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
const childElements = element.children.filter((child) => child instanceof XmlElement);
|
|
131
|
-
const xmlSkippedAnnotation = childElements.find((child) => child.name === "skipped");
|
|
132
|
-
if (xmlSkippedAnnotation)
|
|
133
|
-
annotations.push({ type: "skipped", description: xmlSkippedAnnotation.attributes["message"] });
|
|
134
|
-
const expectedStatus = xmlSkippedAnnotation ? "skipped" : "passed";
|
|
135
|
-
const errors = extractErrors(element);
|
|
136
|
-
const test = {
|
|
137
|
-
title: name,
|
|
138
|
-
location: file && !isNaN(line) ? {
|
|
139
|
-
file,
|
|
140
|
-
line,
|
|
141
|
-
column: 1
|
|
142
|
-
} : FK.NO_LOCATION,
|
|
143
|
-
attempts: [{
|
|
144
|
-
environmentIdx: currentEnvIndex,
|
|
145
|
-
expectedStatus,
|
|
146
|
-
annotations,
|
|
147
|
-
attachments: attachments2,
|
|
148
|
-
startTimestamp,
|
|
149
|
-
duration,
|
|
150
|
-
status: xmlSkippedAnnotation ? "skipped" : errors ? "failed" : "passed",
|
|
151
|
-
errors,
|
|
152
|
-
stdout: extractStdout(element, "system-out"),
|
|
153
|
-
stderr: extractStdout(element, "system-err")
|
|
154
|
-
}]
|
|
155
|
-
};
|
|
156
|
-
currentSuite.tests ??= [];
|
|
157
|
-
currentSuite.tests.push(test);
|
|
158
|
-
}
|
|
159
|
-
context = { ...context, currentEnv, currentEnvIndex, currentSuite, currentTimeMs };
|
|
160
|
-
for (const child of element.children)
|
|
161
|
-
await traverseJUnitReport(context, child);
|
|
162
|
-
}
|
|
163
|
-
async function parseJUnit(xmls, options) {
|
|
164
|
-
const report = {
|
|
165
|
-
category: "junit",
|
|
166
|
-
commitId: options.commitId,
|
|
167
|
-
duration: options.runDuration,
|
|
168
|
-
startTimestamp: options.runStartTimestamp,
|
|
169
|
-
url: options.runUrl,
|
|
170
|
-
environments: [options.defaultEnv],
|
|
171
|
-
suites: [],
|
|
172
|
-
unattributedErrors: []
|
|
173
|
-
};
|
|
174
|
-
const context = {
|
|
175
|
-
currentEnv: options.defaultEnv,
|
|
176
|
-
currentEnvIndex: 0,
|
|
177
|
-
currentTimeMs: 0,
|
|
178
|
-
report,
|
|
179
|
-
currentSuite: void 0,
|
|
180
|
-
attachments: /* @__PURE__ */ new Map(),
|
|
181
|
-
ignoreAttachments: !!options.ignoreAttachments
|
|
182
|
-
};
|
|
183
|
-
for (const xml of xmls) {
|
|
184
|
-
const doc = parseXml(xml);
|
|
185
|
-
for (const element of doc.children)
|
|
186
|
-
await traverseJUnitReport(context, element);
|
|
187
|
-
}
|
|
188
|
-
return {
|
|
189
|
-
report: FK.dedupeSuitesTestsEnvironments(report),
|
|
190
|
-
attachments: Array.from(context.attachments.values())
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
export {
|
|
194
|
-
parseJUnit
|
|
195
|
-
};
|
|
196
|
-
//# sourceMappingURL=junit.js.map
|
package/lib/playwright-test.js
DELETED
|
@@ -1,235 +0,0 @@
|
|
|
1
|
-
import { Multimap } from "@flakiness/shared/common/multimap.js";
|
|
2
|
-
import fs from "fs";
|
|
3
|
-
import path from "path";
|
|
4
|
-
import { FlakinessReport as FK } from "./common/flakinessReport.js";
|
|
5
|
-
import { createTestStepSnippets } from "./createTestStepSnippets.js";
|
|
6
|
-
import { ReportUploader } from "./reportUploader.js";
|
|
7
|
-
import { SystemUtilizationSampler } from "./systemUtilizationSampler.js";
|
|
8
|
-
import { computeGitRoot, createEnvironments, existsAsync, gitCommitInfo, gitFilePath, inferRunUrl, normalizePath, parseDurationMS, sha1Buffer, sha1File, stripAnsi } from "./utils.js";
|
|
9
|
-
class FlakinessReporter {
|
|
10
|
-
constructor(_options = {}) {
|
|
11
|
-
this._options = _options;
|
|
12
|
-
}
|
|
13
|
-
_config;
|
|
14
|
-
_rootSuite;
|
|
15
|
-
_results = new Multimap();
|
|
16
|
-
_unattributedErrors = [];
|
|
17
|
-
_filepathToSteps = new Multimap();
|
|
18
|
-
_systemUtilizationSampler = new SystemUtilizationSampler();
|
|
19
|
-
printsToStdio() {
|
|
20
|
-
return false;
|
|
21
|
-
}
|
|
22
|
-
onBegin(config, suite) {
|
|
23
|
-
this._config = config;
|
|
24
|
-
this._rootSuite = suite;
|
|
25
|
-
}
|
|
26
|
-
onError(error) {
|
|
27
|
-
this._unattributedErrors.push(error);
|
|
28
|
-
}
|
|
29
|
-
onTestBegin(test) {
|
|
30
|
-
}
|
|
31
|
-
onTestEnd(test, result) {
|
|
32
|
-
this._results.set(test, result);
|
|
33
|
-
}
|
|
34
|
-
async _toFKSuites(context, pwSuite) {
|
|
35
|
-
const location = pwSuite.location;
|
|
36
|
-
if (pwSuite.type === "root" || pwSuite.type === "project" || !location)
|
|
37
|
-
return (await Promise.all(pwSuite.suites.map((suite) => this._toFKSuites(context, suite)))).flat();
|
|
38
|
-
let type = "suite";
|
|
39
|
-
if (pwSuite.type === "file")
|
|
40
|
-
type = "file";
|
|
41
|
-
else if (pwSuite.type === "describe" && !pwSuite.title)
|
|
42
|
-
type = "anonymous suite";
|
|
43
|
-
return [{
|
|
44
|
-
type,
|
|
45
|
-
title: pwSuite.title,
|
|
46
|
-
location: this._createLocation(context, location),
|
|
47
|
-
suites: (await Promise.all(pwSuite.suites.map((suite) => this._toFKSuites(context, suite)))).flat(),
|
|
48
|
-
tests: await Promise.all(pwSuite.tests.map((test) => this._toFKTest(context, test)))
|
|
49
|
-
}];
|
|
50
|
-
}
|
|
51
|
-
async _toFKTest(context, pwTest) {
|
|
52
|
-
return {
|
|
53
|
-
title: pwTest.title,
|
|
54
|
-
// Playwright Test tags must start with '@' so we cut it off.
|
|
55
|
-
tags: pwTest.tags.map((tag) => tag.startsWith("@") ? tag.substring(1) : tag),
|
|
56
|
-
location: this._createLocation(context, pwTest.location),
|
|
57
|
-
// de-duplication of tests will happen later, so here we will have all attempts.
|
|
58
|
-
attempts: await Promise.all(this._results.getAll(pwTest).map((result) => this._toFKRunAttempt(context, pwTest, result)))
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
async _toFKRunAttempt(context, pwTest, result) {
|
|
62
|
-
const attachments = [];
|
|
63
|
-
const attempt = {
|
|
64
|
-
timeout: parseDurationMS(pwTest.timeout),
|
|
65
|
-
annotations: pwTest.annotations.map((annotation) => ({
|
|
66
|
-
type: annotation.type,
|
|
67
|
-
description: annotation.description,
|
|
68
|
-
location: annotation.location ? this._createLocation(context, annotation.location) : void 0
|
|
69
|
-
})),
|
|
70
|
-
environmentIdx: context.project2environmentIdx.get(pwTest.parent.project()),
|
|
71
|
-
expectedStatus: pwTest.expectedStatus,
|
|
72
|
-
parallelIndex: result.parallelIndex,
|
|
73
|
-
status: result.status,
|
|
74
|
-
errors: result.errors && result.errors.length ? result.errors.map((error) => this._toFKTestError(context, error)) : void 0,
|
|
75
|
-
stdout: result.stdout ? result.stdout.map(toSTDIOEntry) : void 0,
|
|
76
|
-
stderr: result.stderr ? result.stderr.map(toSTDIOEntry) : void 0,
|
|
77
|
-
steps: result.steps ? result.steps.map((jsonTestStep) => this._toFKTestStep(context, jsonTestStep)) : void 0,
|
|
78
|
-
startTimestamp: +result.startTime,
|
|
79
|
-
duration: +result.duration,
|
|
80
|
-
attachments
|
|
81
|
-
};
|
|
82
|
-
await Promise.all((result.attachments ?? []).map(async (jsonAttachment) => {
|
|
83
|
-
if (jsonAttachment.path && !await existsAsync(jsonAttachment.path)) {
|
|
84
|
-
context.unaccessibleAttachmentPaths.push(jsonAttachment.path);
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
const id = jsonAttachment.path ? await sha1File(jsonAttachment.path) : sha1Buffer(jsonAttachment.body ?? "");
|
|
88
|
-
context.attachments.set(id, {
|
|
89
|
-
contentType: jsonAttachment.contentType,
|
|
90
|
-
id,
|
|
91
|
-
body: jsonAttachment.body,
|
|
92
|
-
path: jsonAttachment.path
|
|
93
|
-
});
|
|
94
|
-
attachments.push({
|
|
95
|
-
id,
|
|
96
|
-
name: jsonAttachment.name,
|
|
97
|
-
contentType: jsonAttachment.contentType
|
|
98
|
-
});
|
|
99
|
-
}));
|
|
100
|
-
return attempt;
|
|
101
|
-
}
|
|
102
|
-
_toFKTestStep(context, pwStep) {
|
|
103
|
-
const step = {
|
|
104
|
-
// NOTE: jsonStep.duration was -1 in some playwright versions
|
|
105
|
-
duration: parseDurationMS(Math.max(pwStep.duration, 0)),
|
|
106
|
-
title: pwStep.title,
|
|
107
|
-
location: pwStep.location ? this._createLocation(context, pwStep.location) : void 0
|
|
108
|
-
};
|
|
109
|
-
if (pwStep.location) {
|
|
110
|
-
const resolvedPath = path.resolve(pwStep.location.file);
|
|
111
|
-
this._filepathToSteps.set(resolvedPath, step);
|
|
112
|
-
}
|
|
113
|
-
if (pwStep.error)
|
|
114
|
-
step.error = this._toFKTestError(context, pwStep.error);
|
|
115
|
-
if (pwStep.steps)
|
|
116
|
-
step.steps = pwStep.steps.map((childJSONStep) => this._toFKTestStep(context, childJSONStep));
|
|
117
|
-
return step;
|
|
118
|
-
}
|
|
119
|
-
_createLocation(context, pwLocation) {
|
|
120
|
-
return {
|
|
121
|
-
file: gitFilePath(context.gitRoot, normalizePath(pwLocation.file)),
|
|
122
|
-
line: pwLocation.line,
|
|
123
|
-
column: pwLocation.column
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
_toFKTestError(context, pwError) {
|
|
127
|
-
return {
|
|
128
|
-
location: pwError.location ? this._createLocation(context, pwError.location) : void 0,
|
|
129
|
-
message: stripAnsi(pwError.message ?? "").split("\n")[0],
|
|
130
|
-
snippet: pwError.snippet,
|
|
131
|
-
stack: pwError.stack,
|
|
132
|
-
value: pwError.value
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
async onEnd(result) {
|
|
136
|
-
this._systemUtilizationSampler.dispose();
|
|
137
|
-
if (!this._config || !this._rootSuite)
|
|
138
|
-
throw new Error("ERROR: failed to resolve config");
|
|
139
|
-
const uploadOptions = ReportUploader.optionsFromEnv({
|
|
140
|
-
flakinessAccessToken: this._options.token,
|
|
141
|
-
flakinessEndpoint: this._options.endpoint
|
|
142
|
-
});
|
|
143
|
-
if (!uploadOptions) {
|
|
144
|
-
console.log(`[flakiness.io] Uploading skipped since no FLAKINESS_ACCESS_TOKEN is specified`);
|
|
145
|
-
return;
|
|
146
|
-
}
|
|
147
|
-
let commitId;
|
|
148
|
-
try {
|
|
149
|
-
commitId = gitCommitInfo(this._config.rootDir);
|
|
150
|
-
} catch (e) {
|
|
151
|
-
console.log(`[flakiness.io] Uploading skipped since failed to get commit info - is this a git repo?`);
|
|
152
|
-
return;
|
|
153
|
-
}
|
|
154
|
-
const gitRoot = normalizePath(computeGitRoot(this._config.rootDir));
|
|
155
|
-
const configPath = this._config.configFile ? gitFilePath(gitRoot, normalizePath(this._config.configFile)) : void 0;
|
|
156
|
-
const context = {
|
|
157
|
-
project2environmentIdx: /* @__PURE__ */ new Map(),
|
|
158
|
-
testBaseDir: normalizePath(this._config.rootDir),
|
|
159
|
-
gitRoot,
|
|
160
|
-
attachments: /* @__PURE__ */ new Map(),
|
|
161
|
-
unaccessibleAttachmentPaths: []
|
|
162
|
-
};
|
|
163
|
-
const environmentsMap = createEnvironments(this._config.projects);
|
|
164
|
-
if (this._options.collectBrowserVersions) {
|
|
165
|
-
try {
|
|
166
|
-
let playwrightPath = fs.realpathSync(process.argv[1]);
|
|
167
|
-
while (path.basename(playwrightPath) !== "test")
|
|
168
|
-
playwrightPath = path.dirname(playwrightPath);
|
|
169
|
-
const module = await import(path.join(playwrightPath, "index.js"));
|
|
170
|
-
for (const [project, env] of environmentsMap) {
|
|
171
|
-
const { browserName = "chromium", channel, headless } = project.use;
|
|
172
|
-
let browserType;
|
|
173
|
-
switch (browserName) {
|
|
174
|
-
case "chromium":
|
|
175
|
-
browserType = module.default.chromium;
|
|
176
|
-
break;
|
|
177
|
-
case "firefox":
|
|
178
|
-
browserType = module.default.firefox;
|
|
179
|
-
break;
|
|
180
|
-
case "webkit":
|
|
181
|
-
browserType = module.default.webkit;
|
|
182
|
-
break;
|
|
183
|
-
default:
|
|
184
|
-
throw new Error(`Unsupported browser: ${browserName}`);
|
|
185
|
-
}
|
|
186
|
-
const browser = await browserType.launch({ channel, headless });
|
|
187
|
-
const version = browser.version();
|
|
188
|
-
await browser.close();
|
|
189
|
-
env.userSuppliedData ??= {};
|
|
190
|
-
env.userSuppliedData["browser"] = (channel ?? browserName).toLowerCase().trim() + " " + version;
|
|
191
|
-
}
|
|
192
|
-
} catch (e) {
|
|
193
|
-
console.error("[flakiness.io] failed to resolve browser version", e);
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
const environments = [...environmentsMap.values()];
|
|
197
|
-
for (let envIdx = 0; envIdx < environments.length; ++envIdx)
|
|
198
|
-
context.project2environmentIdx.set(this._config.projects[envIdx], envIdx);
|
|
199
|
-
const relatedCommitIds = this._options.relatedCommitIds;
|
|
200
|
-
const report = FK.dedupeSuitesTestsEnvironments({
|
|
201
|
-
category: "playwright",
|
|
202
|
-
commitId,
|
|
203
|
-
relatedCommitIds,
|
|
204
|
-
systemUtilization: this._systemUtilizationSampler.result,
|
|
205
|
-
configPath,
|
|
206
|
-
url: inferRunUrl(),
|
|
207
|
-
environments,
|
|
208
|
-
suites: await this._toFKSuites(context, this._rootSuite),
|
|
209
|
-
opaqueData: this._config,
|
|
210
|
-
unattributedErrors: this._unattributedErrors.map((e) => this._toFKTestError(context, e)),
|
|
211
|
-
duration: parseDurationMS(result.duration),
|
|
212
|
-
startTimestamp: +result.startTime
|
|
213
|
-
});
|
|
214
|
-
createTestStepSnippets(this._filepathToSteps);
|
|
215
|
-
for (const unaccessibleAttachment of context.unaccessibleAttachmentPaths)
|
|
216
|
-
console.warn(`[flakiness.io] WARN: cannot access attachment ${unaccessibleAttachment}`);
|
|
217
|
-
const uploadError = await ReportUploader.upload({
|
|
218
|
-
report,
|
|
219
|
-
attachments: [...context.attachments.values()],
|
|
220
|
-
...uploadOptions,
|
|
221
|
-
log: console.log
|
|
222
|
-
});
|
|
223
|
-
if (uploadError)
|
|
224
|
-
return { status: "failed" };
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
function toSTDIOEntry(data) {
|
|
228
|
-
if (Buffer.isBuffer(data))
|
|
229
|
-
return { buffer: data.toString("base64") };
|
|
230
|
-
return { text: data };
|
|
231
|
-
}
|
|
232
|
-
export {
|
|
233
|
-
FlakinessReporter as default
|
|
234
|
-
};
|
|
235
|
-
//# sourceMappingURL=playwright-test.js.map
|
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
import debug from "debug";
|
|
2
|
-
import { posix as posixPath } from "path";
|
|
3
|
-
import { FlakinessReport as FK, FlakinessReport } from "./common/flakinessReport.js";
|
|
4
|
-
import { computeGitRoot, createEnvironments, existsAsync, getOSInfo, gitCommitInfo, gitFilePath, inferRunUrl, normalizePath, parseDurationMS, parseStringDate, sha1Buffer, sha1File, stripAnsi } from "./utils.js";
|
|
5
|
-
const dlog = debug("flakiness:json-report");
|
|
6
|
-
var PlaywrightJSONReport;
|
|
7
|
-
((PlaywrightJSONReport2) => {
|
|
8
|
-
function collectMetadata(somePathInsideProject = process.cwd()) {
|
|
9
|
-
const commitId = gitCommitInfo(somePathInsideProject);
|
|
10
|
-
const osInfo = getOSInfo();
|
|
11
|
-
const metadata = {
|
|
12
|
-
gitRoot: computeGitRoot(somePathInsideProject),
|
|
13
|
-
commitId,
|
|
14
|
-
osName: osInfo.name,
|
|
15
|
-
arch: osInfo.arch,
|
|
16
|
-
osVersion: osInfo.version,
|
|
17
|
-
runURL: inferRunUrl()
|
|
18
|
-
};
|
|
19
|
-
dlog(`metadata directory: ${somePathInsideProject}`);
|
|
20
|
-
dlog(`metadata: ${JSON.stringify(metadata)}`);
|
|
21
|
-
dlog(`commit info: ${JSON.stringify(commitId)}`);
|
|
22
|
-
dlog(`os info: ${JSON.stringify(osInfo)}`);
|
|
23
|
-
return metadata;
|
|
24
|
-
}
|
|
25
|
-
PlaywrightJSONReport2.collectMetadata = collectMetadata;
|
|
26
|
-
async function parse(metadata, jsonReport, options) {
|
|
27
|
-
const context = {
|
|
28
|
-
projectId2environmentIdx: /* @__PURE__ */ new Map(),
|
|
29
|
-
testBaseDir: normalizePath(jsonReport.config.rootDir),
|
|
30
|
-
gitRoot: metadata.gitRoot,
|
|
31
|
-
attachments: /* @__PURE__ */ new Map(),
|
|
32
|
-
unaccessibleAttachmentPaths: [],
|
|
33
|
-
extractAttachments: options.extractAttachments
|
|
34
|
-
};
|
|
35
|
-
const configPath = jsonReport.config.configFile ? gitFilePath(context.gitRoot, normalizePath(jsonReport.config.configFile)) : void 0;
|
|
36
|
-
const report = {
|
|
37
|
-
category: FlakinessReport.CATEGORY_PLAYWRIGHT,
|
|
38
|
-
commitId: metadata.commitId,
|
|
39
|
-
configPath,
|
|
40
|
-
url: metadata.runURL,
|
|
41
|
-
environments: [],
|
|
42
|
-
suites: [],
|
|
43
|
-
opaqueData: jsonReport.config,
|
|
44
|
-
unattributedErrors: jsonReport.errors.map((error) => parseJSONError(context, error)),
|
|
45
|
-
// The report.stats is a releatively new addition to Playwright's JSONReport,
|
|
46
|
-
// so we have to polyfill with some reasonable values when it's missing.
|
|
47
|
-
duration: jsonReport.stats?.duration && jsonReport.stats?.duration > 0 ? parseDurationMS(jsonReport.stats.duration) : 0,
|
|
48
|
-
startTimestamp: jsonReport.stats && jsonReport.stats.startTime ? parseStringDate(jsonReport.stats.startTime) : Date.now()
|
|
49
|
-
};
|
|
50
|
-
report.environments = [...createEnvironments(jsonReport.config.projects).values()];
|
|
51
|
-
for (let envIdx = 0; envIdx < report.environments.length; ++envIdx)
|
|
52
|
-
context.projectId2environmentIdx.set(jsonReport.config.projects[envIdx].id, envIdx);
|
|
53
|
-
report.suites = await Promise.all(jsonReport.suites.map((suite) => parseJSONSuite(context, suite)));
|
|
54
|
-
return {
|
|
55
|
-
report: FK.dedupeSuitesTestsEnvironments(report),
|
|
56
|
-
attachments: [...context.attachments.values()],
|
|
57
|
-
unaccessibleAttachmentPaths: context.unaccessibleAttachmentPaths
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
PlaywrightJSONReport2.parse = parse;
|
|
61
|
-
})(PlaywrightJSONReport || (PlaywrightJSONReport = {}));
|
|
62
|
-
async function parseJSONSuite(context, jsonSuite) {
|
|
63
|
-
let type = "suite";
|
|
64
|
-
if (jsonSuite.column === 0 && jsonSuite.line === 0)
|
|
65
|
-
type = "file";
|
|
66
|
-
else if (!jsonSuite.title)
|
|
67
|
-
type = "anonymous suite";
|
|
68
|
-
const suite = {
|
|
69
|
-
type,
|
|
70
|
-
title: jsonSuite.title,
|
|
71
|
-
location: {
|
|
72
|
-
file: gitFilePath(context.gitRoot, normalizePath(jsonSuite.file)),
|
|
73
|
-
line: jsonSuite.line,
|
|
74
|
-
column: jsonSuite.column
|
|
75
|
-
}
|
|
76
|
-
};
|
|
77
|
-
if (jsonSuite.suites && jsonSuite.suites.length)
|
|
78
|
-
suite.suites = await Promise.all(jsonSuite.suites.map((suite2) => parseJSONSuite(context, suite2)));
|
|
79
|
-
if (jsonSuite.specs && jsonSuite.specs.length)
|
|
80
|
-
suite.tests = await Promise.all(jsonSuite.specs.map((spec) => parseJSONSpec(context, spec)));
|
|
81
|
-
return suite;
|
|
82
|
-
}
|
|
83
|
-
async function parseJSONSpec(context, jsonSpec) {
|
|
84
|
-
const test = {
|
|
85
|
-
title: jsonSpec.title,
|
|
86
|
-
tags: jsonSpec.tags,
|
|
87
|
-
location: {
|
|
88
|
-
file: gitFilePath(context.gitRoot, normalizePath(posixPath.join(context.testBaseDir, normalizePath(jsonSpec.file)))),
|
|
89
|
-
line: jsonSpec.line,
|
|
90
|
-
column: jsonSpec.column
|
|
91
|
-
},
|
|
92
|
-
attempts: []
|
|
93
|
-
};
|
|
94
|
-
for (const jsonTest of jsonSpec.tests) {
|
|
95
|
-
const environmentIdx = context.projectId2environmentIdx.get(jsonTest.projectId);
|
|
96
|
-
if (environmentIdx === void 0)
|
|
97
|
-
throw new Error("Inconsistent report - no project for a test found!");
|
|
98
|
-
const testResults = jsonTest.results.filter((result) => result.status !== void 0);
|
|
99
|
-
if (!testResults.length)
|
|
100
|
-
continue;
|
|
101
|
-
test.attempts.push(...await Promise.all(testResults.map((jsonTestResult) => parseJSONTestResult(context, jsonTest, environmentIdx, jsonTestResult))));
|
|
102
|
-
}
|
|
103
|
-
return test;
|
|
104
|
-
}
|
|
105
|
-
function createLocation(context, location) {
|
|
106
|
-
return {
|
|
107
|
-
file: gitFilePath(context.gitRoot, normalizePath(location.file)),
|
|
108
|
-
line: location.line,
|
|
109
|
-
column: location.column
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
async function parseJSONTestResult(context, jsonTest, environmentIdx, jsonTestResult) {
|
|
113
|
-
const attachments = [];
|
|
114
|
-
const attempt = {
|
|
115
|
-
timeout: parseDurationMS(jsonTest.timeout),
|
|
116
|
-
annotations: jsonTest.annotations.map((annotation) => ({
|
|
117
|
-
type: annotation.type,
|
|
118
|
-
description: annotation.description,
|
|
119
|
-
location: annotation.location ? createLocation(context, annotation.location) : void 0
|
|
120
|
-
})),
|
|
121
|
-
environmentIdx,
|
|
122
|
-
expectedStatus: jsonTest.expectedStatus,
|
|
123
|
-
parallelIndex: jsonTestResult.parallelIndex,
|
|
124
|
-
status: jsonTestResult.status,
|
|
125
|
-
errors: jsonTestResult.errors && jsonTestResult.errors.length ? jsonTestResult.errors.map((error) => parseJSONError(context, error)) : void 0,
|
|
126
|
-
stdout: jsonTestResult.stdout && jsonTestResult.stdout.length ? jsonTestResult.stdout : void 0,
|
|
127
|
-
stderr: jsonTestResult.stderr && jsonTestResult.stderr.length ? jsonTestResult.stderr : void 0,
|
|
128
|
-
steps: jsonTestResult.steps ? jsonTestResult.steps.map((jsonTestStep) => parseJSONTestStep(context, jsonTestStep)) : void 0,
|
|
129
|
-
startTimestamp: parseStringDate(jsonTestResult.startTime),
|
|
130
|
-
duration: jsonTestResult.duration && jsonTestResult.duration > 0 ? parseDurationMS(jsonTestResult.duration) : 0,
|
|
131
|
-
attachments
|
|
132
|
-
};
|
|
133
|
-
if (context.extractAttachments) {
|
|
134
|
-
await Promise.all((jsonTestResult.attachments ?? []).map(async (jsonAttachment) => {
|
|
135
|
-
if (jsonAttachment.path && !await existsAsync(jsonAttachment.path)) {
|
|
136
|
-
context.unaccessibleAttachmentPaths.push(jsonAttachment.path);
|
|
137
|
-
return;
|
|
138
|
-
}
|
|
139
|
-
const id = jsonAttachment.path ? await sha1File(jsonAttachment.path) : sha1Buffer(jsonAttachment.body ?? "");
|
|
140
|
-
context.attachments.set(id, {
|
|
141
|
-
contentType: jsonAttachment.contentType,
|
|
142
|
-
id,
|
|
143
|
-
body: jsonAttachment.body ? Buffer.from(jsonAttachment.body) : void 0,
|
|
144
|
-
path: jsonAttachment.path
|
|
145
|
-
});
|
|
146
|
-
attachments.push({
|
|
147
|
-
id,
|
|
148
|
-
name: jsonAttachment.name,
|
|
149
|
-
contentType: jsonAttachment.contentType
|
|
150
|
-
});
|
|
151
|
-
}));
|
|
152
|
-
}
|
|
153
|
-
return attempt;
|
|
154
|
-
}
|
|
155
|
-
function parseJSONTestStep(context, jsonStep) {
|
|
156
|
-
const step = {
|
|
157
|
-
// NOTE: jsonStep.duration was -1 in some playwright versions
|
|
158
|
-
duration: parseDurationMS(Math.max(jsonStep.duration, 0)),
|
|
159
|
-
title: jsonStep.title
|
|
160
|
-
};
|
|
161
|
-
if (jsonStep.error)
|
|
162
|
-
step.error = parseJSONError(context, jsonStep.error);
|
|
163
|
-
if (jsonStep.steps)
|
|
164
|
-
step.steps = jsonStep.steps.map((childJSONStep) => parseJSONTestStep(context, childJSONStep));
|
|
165
|
-
return step;
|
|
166
|
-
}
|
|
167
|
-
function parseJSONError(context, error) {
|
|
168
|
-
return {
|
|
169
|
-
location: error.location ? createLocation(context, error.location) : void 0,
|
|
170
|
-
message: error.message ? stripAnsi(error.message).split("\n")[0] : void 0,
|
|
171
|
-
stack: error.stack,
|
|
172
|
-
value: error.value
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
export {
|
|
176
|
-
PlaywrightJSONReport
|
|
177
|
-
};
|
|
178
|
-
//# sourceMappingURL=playwrightJSONReport.js.map
|