@alwaysmeticulous/cli 2.39.0 → 2.40.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/test-run.api.d.ts +3 -31
- package/dist/api/test-run.api.js +3 -96
- package/dist/commands/download-replay/download-replay.command.js +4 -4
- package/dist/commands/download-session/download-session.command.js +4 -4
- package/dist/commands/record/record.command.js +8 -11
- package/dist/commands/replay/replay.command.d.ts +4 -20
- package/dist/commands/replay/replay.command.js +4 -239
- package/dist/commands/run-all-tests/run-all-tests.command.js +5 -6
- package/dist/commands/show-project/show-project.command.js +2 -3
- package/dist/index.d.ts +0 -7
- package/dist/index.js +1 -13
- package/dist/main.js +5 -11
- package/dist/utils/sentry.utils.d.ts +0 -4
- package/dist/utils/sentry.utils.js +3 -23
- package/package.json +14 -32
- package/dist/api/client.d.ts +0 -5
- package/dist/api/client.js +0 -32
- package/dist/api/download.d.ts +0 -1
- package/dist/api/download.js +0 -21
- package/dist/api/project.api.d.ts +0 -3
- package/dist/api/project.api.js +0 -22
- package/dist/api/replay-diff.api.d.ts +0 -13
- package/dist/api/replay-diff.api.js +0 -32
- package/dist/api/replay.api.d.ts +0 -36
- package/dist/api/replay.api.js +0 -89
- package/dist/api/session.api.d.ts +0 -6
- package/dist/api/session.api.js +0 -45
- package/dist/api/types.d.ts +0 -39
- package/dist/api/types.js +0 -2
- package/dist/api/upload.d.ts +0 -1
- package/dist/api/upload.js +0 -18
- package/dist/archive/archive.d.ts +0 -4
- package/dist/archive/archive.js +0 -64
- package/dist/command-utils/common-types.d.ts +0 -20
- package/dist/command-utils/common-types.js +0 -2
- package/dist/commands/bootstrap/bootstrap.command.d.ts +0 -2
- package/dist/commands/bootstrap/bootstrap.command.js +0 -33
- package/dist/commands/create-test/create-test.command.d.ts +0 -94
- package/dist/commands/create-test/create-test.command.js +0 -170
- package/dist/commands/replay/utils/compute-diff.d.ts +0 -13
- package/dist/commands/replay/utils/compute-diff.js +0 -95
- package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.d.ts +0 -1
- package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.js +0 -27
- package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts +0 -53
- package/dist/commands/screenshot-diff/screenshot-diff.command.js +0 -182
- package/dist/commands/screenshot-diff/utils/get-screenshot-filename.d.ts +0 -2
- package/dist/commands/screenshot-diff/utils/get-screenshot-filename.js +0 -18
- package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.d.ts +0 -2
- package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.js +0 -24
- package/dist/commands/screenshot-diff/utils/has-notable-differences.d.ts +0 -2
- package/dist/commands/screenshot-diff/utils/has-notable-differences.js +0 -10
- package/dist/commands/serve/serve.command.d.ts +0 -10
- package/dist/commands/serve/serve.command.js +0 -31
- package/dist/commands/update-tests/update-tests.command.d.ts +0 -21
- package/dist/commands/update-tests/update-tests.command.js +0 -96
- package/dist/config/config.d.ts +0 -3
- package/dist/config/config.js +0 -70
- package/dist/config/config.types.d.ts +0 -16
- package/dist/config/config.types.js +0 -2
- package/dist/config/snippets.d.ts +0 -1
- package/dist/config/snippets.js +0 -18
- package/dist/errors/config.d.ts +0 -3
- package/dist/errors/config.js +0 -10
- package/dist/image/diff.utils.d.ts +0 -19
- package/dist/image/diff.utils.js +0 -25
- package/dist/image/io.utils.d.ts +0 -3
- package/dist/image/io.utils.js +0 -25
- package/dist/local-data/local-data.utils.d.ts +0 -20
- package/dist/local-data/local-data.utils.js +0 -92
- package/dist/local-data/replay-assets.d.ts +0 -3
- package/dist/local-data/replay-assets.js +0 -94
- package/dist/local-data/replays.d.ts +0 -18
- package/dist/local-data/replays.js +0 -87
- package/dist/local-data/screenshot-diffs.d.ts +0 -7
- package/dist/local-data/screenshot-diffs.js +0 -20
- package/dist/local-data/serve-assets-from-simulation.d.ts +0 -5
- package/dist/local-data/serve-assets-from-simulation.js +0 -52
- package/dist/local-data/sessions.d.ts +0 -10
- package/dist/local-data/sessions.js +0 -41
- package/dist/parallel-tests/__tests__/merge-test-results.spec.d.ts +0 -1
- package/dist/parallel-tests/__tests__/merge-test-results.spec.js +0 -104
- package/dist/parallel-tests/__tests__/mock-test-results.d.ts +0 -17
- package/dist/parallel-tests/__tests__/mock-test-results.js +0 -106
- package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.d.ts +0 -1
- package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.js +0 -205
- package/dist/parallel-tests/execute-test-in-child-process.d.ts +0 -3
- package/dist/parallel-tests/execute-test-in-child-process.js +0 -44
- package/dist/parallel-tests/merge-test-results.d.ts +0 -12
- package/dist/parallel-tests/merge-test-results.js +0 -122
- package/dist/parallel-tests/messages.types.d.ts +0 -17
- package/dist/parallel-tests/messages.types.js +0 -2
- package/dist/parallel-tests/parallel-replay.handler.d.ts +0 -3
- package/dist/parallel-tests/parallel-replay.handler.js +0 -57
- package/dist/parallel-tests/parallel-replay.types.d.ts +0 -16
- package/dist/parallel-tests/parallel-replay.types.js +0 -2
- package/dist/parallel-tests/parallel-tests.handler.d.ts +0 -14
- package/dist/parallel-tests/parallel-tests.handler.js +0 -194
- package/dist/parallel-tests/run-all-tests.d.ts +0 -73
- package/dist/parallel-tests/run-all-tests.js +0 -237
- package/dist/parallel-tests/run-all-tests.types.d.ts +0 -6
- package/dist/parallel-tests/run-all-tests.types.js +0 -2
- package/dist/parallel-tests/screenshot-diff-results.utils.d.ts +0 -7
- package/dist/parallel-tests/screenshot-diff-results.utils.js +0 -20
- package/dist/parallel-tests/task.handler.d.ts +0 -1
- package/dist/parallel-tests/task.handler.js +0 -86
- package/dist/utils/api-token.utils.d.ts +0 -1
- package/dist/utils/api-token.utils.js +0 -31
- package/dist/utils/commit-sha.utils.d.ts +0 -1
- package/dist/utils/commit-sha.utils.js +0 -42
- package/dist/utils/config.utils.d.ts +0 -7
- package/dist/utils/config.utils.js +0 -33
- package/dist/utils/github-summary.utils.d.ts +0 -5
- package/dist/utils/github-summary.utils.js +0 -38
- package/dist/utils/run-all-tests.utils.d.ts +0 -16
- package/dist/utils/run-all-tests.utils.js +0 -32
- package/dist/utils/test-run-environment.utils.d.ts +0 -2
- package/dist/utils/test-run-environment.utils.js +0 -25
- package/dist/utils/version.utils.d.ts +0 -1
- package/dist/utils/version.utils.js +0 -17
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.differentSize = exports.missingBaseAndHead = exports.missingHead = exports.missingBase = exports.flake = exports.noDiff = exports.diff = exports.testResult = exports.id = void 0;
|
|
4
|
-
const id = (eventNumber = 0) => ({
|
|
5
|
-
type: "after-event",
|
|
6
|
-
eventNumber,
|
|
7
|
-
});
|
|
8
|
-
exports.id = id;
|
|
9
|
-
const testResult = (result, screenshotDiffResults, testCase) => {
|
|
10
|
-
var _a;
|
|
11
|
-
return {
|
|
12
|
-
...testCase,
|
|
13
|
-
sessionId: (_a = testCase === null || testCase === void 0 ? void 0 : testCase.sessionId) !== null && _a !== void 0 ? _a : "mock-session-id",
|
|
14
|
-
headReplayId: "mock-head-replay-id",
|
|
15
|
-
result,
|
|
16
|
-
screenshotDiffResultsByBaseReplayId: {
|
|
17
|
-
"mock-base-replay-id": screenshotDiffResults,
|
|
18
|
-
},
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
exports.testResult = testResult;
|
|
22
|
-
const diff = (eventNumber) => {
|
|
23
|
-
return {
|
|
24
|
-
identifier: (0, exports.id)(eventNumber),
|
|
25
|
-
outcome: "diff",
|
|
26
|
-
headScreenshotFile: "mock-head-file",
|
|
27
|
-
baseScreenshotFile: "mock-base-file",
|
|
28
|
-
width: 1920,
|
|
29
|
-
height: 1080,
|
|
30
|
-
mismatchFraction: 0.01,
|
|
31
|
-
mismatchPixels: 1000,
|
|
32
|
-
};
|
|
33
|
-
};
|
|
34
|
-
exports.diff = diff;
|
|
35
|
-
const noDiff = (eventNumber) => {
|
|
36
|
-
return {
|
|
37
|
-
identifier: (0, exports.id)(eventNumber),
|
|
38
|
-
outcome: "no-diff",
|
|
39
|
-
headScreenshotFile: "mock-head-file",
|
|
40
|
-
baseScreenshotFile: "mock-base-file",
|
|
41
|
-
width: 1920,
|
|
42
|
-
height: 1080,
|
|
43
|
-
mismatchFraction: 0.01,
|
|
44
|
-
mismatchPixels: 1000,
|
|
45
|
-
};
|
|
46
|
-
};
|
|
47
|
-
exports.noDiff = noDiff;
|
|
48
|
-
const flake = (eventNumber, diffToBaseScreenshot, diffsToHeadScreenshotOnRetries) => {
|
|
49
|
-
return {
|
|
50
|
-
identifier: (0, exports.id)(eventNumber),
|
|
51
|
-
outcome: "flake",
|
|
52
|
-
diffToBaseScreenshot: asSingleTryDiff(diffToBaseScreenshot),
|
|
53
|
-
diffsToHeadScreenshotOnRetries: diffsToHeadScreenshotOnRetries.map(asRetryDiff),
|
|
54
|
-
};
|
|
55
|
-
};
|
|
56
|
-
exports.flake = flake;
|
|
57
|
-
const missingBase = (eventNumber) => {
|
|
58
|
-
return {
|
|
59
|
-
identifier: (0, exports.id)(eventNumber),
|
|
60
|
-
outcome: "missing-base",
|
|
61
|
-
headScreenshotFile: "mock-head-file",
|
|
62
|
-
};
|
|
63
|
-
};
|
|
64
|
-
exports.missingBase = missingBase;
|
|
65
|
-
const missingHead = (eventNumber) => {
|
|
66
|
-
return {
|
|
67
|
-
identifier: (0, exports.id)(eventNumber),
|
|
68
|
-
outcome: "missing-head",
|
|
69
|
-
baseScreenshotFile: "mock-base-file",
|
|
70
|
-
};
|
|
71
|
-
};
|
|
72
|
-
exports.missingHead = missingHead;
|
|
73
|
-
const missingBaseAndHead = (eventNumber) => {
|
|
74
|
-
return {
|
|
75
|
-
identifier: (0, exports.id)(eventNumber),
|
|
76
|
-
outcome: "missing-base-and-head",
|
|
77
|
-
};
|
|
78
|
-
};
|
|
79
|
-
exports.missingBaseAndHead = missingBaseAndHead;
|
|
80
|
-
const differentSize = (eventNumber) => {
|
|
81
|
-
return {
|
|
82
|
-
identifier: (0, exports.id)(eventNumber),
|
|
83
|
-
outcome: "different-size",
|
|
84
|
-
baseScreenshotFile: "mock-base-file",
|
|
85
|
-
headScreenshotFile: "mock-head-file",
|
|
86
|
-
baseWidth: 1920,
|
|
87
|
-
baseHeight: 1080,
|
|
88
|
-
headWidth: 10,
|
|
89
|
-
headHeight: 5,
|
|
90
|
-
};
|
|
91
|
-
};
|
|
92
|
-
exports.differentSize = differentSize;
|
|
93
|
-
const asSingleTryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
94
|
-
...rest }) => {
|
|
95
|
-
if (rest.outcome === "flake") {
|
|
96
|
-
throw new Error("Must not be a diff with a flake outcome");
|
|
97
|
-
}
|
|
98
|
-
return rest;
|
|
99
|
-
};
|
|
100
|
-
const asRetryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
101
|
-
...rest }) => {
|
|
102
|
-
if (rest.outcome === "flake") {
|
|
103
|
-
throw new Error("Must not be a diff with a flake outcome");
|
|
104
|
-
}
|
|
105
|
-
return rest;
|
|
106
|
-
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const common_1 = require("@alwaysmeticulous/common");
|
|
4
|
-
const parallel_tests_handler_1 = require("../parallel-tests.handler");
|
|
5
|
-
const mock_test_results_1 = require("./mock-test-results");
|
|
6
|
-
describe("runAllTestsInParallel", () => {
|
|
7
|
-
it("maximises the number of tests run in parralel, up to the provided limit", async () => {
|
|
8
|
-
let progress = {
|
|
9
|
-
runningTestCases: 4,
|
|
10
|
-
failedTestCases: 0,
|
|
11
|
-
flakedTestCases: 0,
|
|
12
|
-
passedTestCases: 0,
|
|
13
|
-
};
|
|
14
|
-
let testsStarted = 0;
|
|
15
|
-
const testRunPromises = [(0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)()];
|
|
16
|
-
const overallPromise = (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
17
|
-
testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
|
|
18
|
-
parallelTasks: 2,
|
|
19
|
-
maxRetriesOnFailure: 0,
|
|
20
|
-
rerunTestsNTimes: 0,
|
|
21
|
-
executeTest: async (testCase) => {
|
|
22
|
-
var _a;
|
|
23
|
-
testsStarted++;
|
|
24
|
-
await testRunPromises[parseInt((_a = testCase.title) !== null && _a !== void 0 ? _a : "")].promise;
|
|
25
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase);
|
|
26
|
-
},
|
|
27
|
-
onTestFinished: async (newProgress) => {
|
|
28
|
-
progress = newProgress;
|
|
29
|
-
},
|
|
30
|
-
});
|
|
31
|
-
await waitForPromisesToFlush();
|
|
32
|
-
expect(testsStarted).toEqual(2);
|
|
33
|
-
expect(progress).toEqual({
|
|
34
|
-
runningTestCases: 4,
|
|
35
|
-
failedTestCases: 0,
|
|
36
|
-
flakedTestCases: 0,
|
|
37
|
-
passedTestCases: 0,
|
|
38
|
-
});
|
|
39
|
-
testRunPromises[0].resolve();
|
|
40
|
-
await waitForPromisesToFlush();
|
|
41
|
-
expect(testsStarted).toEqual(3);
|
|
42
|
-
expect(progress).toEqual({
|
|
43
|
-
runningTestCases: 3,
|
|
44
|
-
failedTestCases: 0,
|
|
45
|
-
flakedTestCases: 0,
|
|
46
|
-
passedTestCases: 1,
|
|
47
|
-
});
|
|
48
|
-
testRunPromises[1].resolve();
|
|
49
|
-
testRunPromises[2].resolve();
|
|
50
|
-
await waitForPromisesToFlush();
|
|
51
|
-
expect(testsStarted).toEqual(4);
|
|
52
|
-
expect(progress).toEqual({
|
|
53
|
-
runningTestCases: 1,
|
|
54
|
-
failedTestCases: 0,
|
|
55
|
-
flakedTestCases: 0,
|
|
56
|
-
passedTestCases: 3,
|
|
57
|
-
});
|
|
58
|
-
await expectPromiseToNotHaveResolved(overallPromise);
|
|
59
|
-
testRunPromises[3].resolve();
|
|
60
|
-
await waitForPromisesToFlush();
|
|
61
|
-
expect(progress).toEqual({
|
|
62
|
-
runningTestCases: 0,
|
|
63
|
-
failedTestCases: 0,
|
|
64
|
-
flakedTestCases: 0,
|
|
65
|
-
passedTestCases: 4,
|
|
66
|
-
});
|
|
67
|
-
await overallPromise;
|
|
68
|
-
});
|
|
69
|
-
it("can handle a test failing to run", async () => {
|
|
70
|
-
let progress = {
|
|
71
|
-
runningTestCases: 4,
|
|
72
|
-
failedTestCases: 0,
|
|
73
|
-
flakedTestCases: 0,
|
|
74
|
-
passedTestCases: 0,
|
|
75
|
-
};
|
|
76
|
-
const testRunPromises = [(0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)()];
|
|
77
|
-
const overallPromise = (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
78
|
-
testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
|
|
79
|
-
parallelTasks: 2,
|
|
80
|
-
maxRetriesOnFailure: 0,
|
|
81
|
-
rerunTestsNTimes: 0,
|
|
82
|
-
executeTest: async (testCase) => {
|
|
83
|
-
var _a;
|
|
84
|
-
await testRunPromises[parseInt((_a = testCase.title) !== null && _a !== void 0 ? _a : "")].promise;
|
|
85
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase);
|
|
86
|
-
},
|
|
87
|
-
onTestFinished: async (newProgress) => {
|
|
88
|
-
progress = newProgress;
|
|
89
|
-
},
|
|
90
|
-
});
|
|
91
|
-
testRunPromises[0].resolve();
|
|
92
|
-
testRunPromises[1].reject();
|
|
93
|
-
testRunPromises[2].resolve();
|
|
94
|
-
testRunPromises[3].resolve();
|
|
95
|
-
const result = await overallPromise;
|
|
96
|
-
expect(progress).toEqual({
|
|
97
|
-
runningTestCases: 0,
|
|
98
|
-
failedTestCases: 1,
|
|
99
|
-
flakedTestCases: 0,
|
|
100
|
-
passedTestCases: 3,
|
|
101
|
-
});
|
|
102
|
-
// No result given for test 1 (it can't, because it doesn't have a headReplayId)
|
|
103
|
-
expect(result).toEqual([
|
|
104
|
-
(0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(0)),
|
|
105
|
-
(0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(2)),
|
|
106
|
-
(0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(3)),
|
|
107
|
-
]);
|
|
108
|
-
});
|
|
109
|
-
it("does not retry failed tests if maxRetriesOnFailure is 0", async () => {
|
|
110
|
-
const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
111
|
-
testsToRun: [testCase(0)],
|
|
112
|
-
parallelTasks: 2,
|
|
113
|
-
maxRetriesOnFailure: 0,
|
|
114
|
-
rerunTestsNTimes: 0,
|
|
115
|
-
executeTest: async (testCase) => {
|
|
116
|
-
return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0)], testCase);
|
|
117
|
-
},
|
|
118
|
-
});
|
|
119
|
-
expect(results).toEqual([(0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0)], testCase(0))]);
|
|
120
|
-
});
|
|
121
|
-
it("retries failed tests until maxRetriesOnFailure reached or first flake seen for all screenshots if maxRetriesOnFailure > 0", async () => {
|
|
122
|
-
let retryNum = 0;
|
|
123
|
-
const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
124
|
-
testsToRun: [testCase(0)],
|
|
125
|
-
parallelTasks: 2,
|
|
126
|
-
maxRetriesOnFailure: 10,
|
|
127
|
-
rerunTestsNTimes: 0,
|
|
128
|
-
executeTest: async (testCase, isRetry) => {
|
|
129
|
-
if (!isRetry) {
|
|
130
|
-
return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
131
|
-
}
|
|
132
|
-
if (retryNum === 3) {
|
|
133
|
-
return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
134
|
-
}
|
|
135
|
-
retryNum++;
|
|
136
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
137
|
-
},
|
|
138
|
-
});
|
|
139
|
-
expect(retryNum).toEqual(3);
|
|
140
|
-
expect(results).toEqual([
|
|
141
|
-
(0, mock_test_results_1.testResult)("flake", [
|
|
142
|
-
(0, mock_test_results_1.flake)(0, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)()]),
|
|
143
|
-
(0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()]),
|
|
144
|
-
], testCase(0)),
|
|
145
|
-
]);
|
|
146
|
-
});
|
|
147
|
-
it("marks test as a failure if hit max number of retries and no flakes detected", async () => {
|
|
148
|
-
const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
149
|
-
testsToRun: [testCase(0)],
|
|
150
|
-
parallelTasks: 2,
|
|
151
|
-
maxRetriesOnFailure: 3,
|
|
152
|
-
rerunTestsNTimes: 0,
|
|
153
|
-
executeTest: async (testCase, isRetry) => {
|
|
154
|
-
if (!isRetry) {
|
|
155
|
-
return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
156
|
-
}
|
|
157
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
158
|
-
},
|
|
159
|
-
});
|
|
160
|
-
expect(results).toEqual([
|
|
161
|
-
(0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()])], testCase(0)),
|
|
162
|
-
]);
|
|
163
|
-
});
|
|
164
|
-
it("retries N times if rerunTestsNTimes > 0", async () => {
|
|
165
|
-
let rerunNum = 0;
|
|
166
|
-
const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
|
|
167
|
-
testsToRun: [testCase(0)],
|
|
168
|
-
parallelTasks: 2,
|
|
169
|
-
maxRetriesOnFailure: 0,
|
|
170
|
-
rerunTestsNTimes: 3,
|
|
171
|
-
executeTest: async (testCase, isRetry) => {
|
|
172
|
-
if (!isRetry) {
|
|
173
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.noDiff)(1)], testCase);
|
|
174
|
-
}
|
|
175
|
-
if (rerunNum === 2) {
|
|
176
|
-
return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0), (0, mock_test_results_1.diff)(1)], testCase);
|
|
177
|
-
}
|
|
178
|
-
rerunNum++;
|
|
179
|
-
return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)], testCase);
|
|
180
|
-
},
|
|
181
|
-
});
|
|
182
|
-
expect(rerunNum).toEqual(2);
|
|
183
|
-
expect(results).toEqual([
|
|
184
|
-
(0, mock_test_results_1.testResult)(
|
|
185
|
-
// Flake outcome is returned iff all screenshots are flaky.
|
|
186
|
-
"flake", [(0, mock_test_results_1.flake)(0, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)()]), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.noDiff)(), [(0, mock_test_results_1.diff)()])], testCase(0)),
|
|
187
|
-
]);
|
|
188
|
-
});
|
|
189
|
-
});
|
|
190
|
-
const testCase = (num) => ({
|
|
191
|
-
sessionId: "mock-session-id",
|
|
192
|
-
title: `${num}`,
|
|
193
|
-
baseTestRunId: "mock-base-test-run-id",
|
|
194
|
-
});
|
|
195
|
-
const expectPromiseToNotHaveResolved = async (promise) => {
|
|
196
|
-
let resolved = false;
|
|
197
|
-
promise.then(() => {
|
|
198
|
-
resolved = true;
|
|
199
|
-
}, () => {
|
|
200
|
-
resolved = true;
|
|
201
|
-
});
|
|
202
|
-
await waitForPromisesToFlush();
|
|
203
|
-
expect(resolved).toEqual(false);
|
|
204
|
-
};
|
|
205
|
-
const waitForPromisesToFlush = () => new Promise((resolve) => setTimeout(resolve, 0));
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.executeTestInChildProcess = void 0;
|
|
7
|
-
const child_process_1 = require("child_process");
|
|
8
|
-
const path_1 = require("path");
|
|
9
|
-
const common_1 = require("@alwaysmeticulous/common");
|
|
10
|
-
const loglevel_1 = __importDefault(require("loglevel"));
|
|
11
|
-
const executeTestInChildProcess = (initMessage) => {
|
|
12
|
-
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
13
|
-
const isTypeScript = __filename.endsWith(".ts");
|
|
14
|
-
const taskHandler = (0, path_1.join)(__dirname, `task.handler${isTypeScript ? ".ts" : ".js"}`);
|
|
15
|
-
const deferredResult = (0, common_1.defer)();
|
|
16
|
-
const child = (0, child_process_1.fork)(taskHandler, [], { stdio: "inherit" });
|
|
17
|
-
const messageHandler = (message) => {
|
|
18
|
-
if (message &&
|
|
19
|
-
typeof message === "object" &&
|
|
20
|
-
message["kind"] === "result") {
|
|
21
|
-
const resultMessage = message;
|
|
22
|
-
deferredResult.resolve(resultMessage.data.result);
|
|
23
|
-
child.off("message", messageHandler);
|
|
24
|
-
}
|
|
25
|
-
};
|
|
26
|
-
child.on("error", (error) => {
|
|
27
|
-
if (deferredResult.getState() === "pending") {
|
|
28
|
-
deferredResult.reject(error);
|
|
29
|
-
}
|
|
30
|
-
});
|
|
31
|
-
child.on("exit", (code) => {
|
|
32
|
-
if (code) {
|
|
33
|
-
logger.debug(`child exited with code: ${code}`);
|
|
34
|
-
}
|
|
35
|
-
if (deferredResult.getState() === "pending") {
|
|
36
|
-
deferredResult.reject(new Error("No result"));
|
|
37
|
-
}
|
|
38
|
-
});
|
|
39
|
-
child.on("message", messageHandler);
|
|
40
|
-
// Send test case and arguments to child process
|
|
41
|
-
child.send(initMessage);
|
|
42
|
-
return deferredResult.promise;
|
|
43
|
-
};
|
|
44
|
-
exports.executeTestInChildProcess = executeTestInChildProcess;
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
|
|
2
|
-
import { DetailedTestCaseResult, TestCaseResult } from "../config/config.types";
|
|
3
|
-
export interface ResultsToMerge {
|
|
4
|
-
currentResult: DetailedTestCaseResult;
|
|
5
|
-
/**
|
|
6
|
-
* The result of replaying the session once more against the head commit and
|
|
7
|
-
* comparing the screenshots to those taken on the first replay against the head commit.
|
|
8
|
-
*/
|
|
9
|
-
comparisonToHeadReplay: DetailedTestCaseResult;
|
|
10
|
-
}
|
|
11
|
-
export declare const mergeResults: ({ currentResult, comparisonToHeadReplay, }: ResultsToMerge) => DetailedTestCaseResult;
|
|
12
|
-
export declare const testRunOutcomeFromDiffResults: (currentResult: TestCaseResult["result"], newScreenshotDiffResults: ScreenshotDiffResult[]) => TestCaseResult["result"];
|
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.testRunOutcomeFromDiffResults = exports.mergeResults = void 0;
|
|
7
|
-
const utils_1 = require("@sentry/utils");
|
|
8
|
-
const fast_json_stable_stringify_1 = __importDefault(require("fast-json-stable-stringify"));
|
|
9
|
-
const has_notable_differences_1 = require("../commands/screenshot-diff/utils/has-notable-differences");
|
|
10
|
-
const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
|
|
11
|
-
const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
|
|
12
|
-
// If any of the screenshots diffs in comparisonToHeadReplay show a diff against one
|
|
13
|
-
// of the screenshots that orignally failed in currentResult then we have a flake
|
|
14
|
-
// on our hands
|
|
15
|
-
const retryDiffById = new Map();
|
|
16
|
-
(0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(comparisonToHeadReplay).forEach((result) => {
|
|
17
|
-
const hash = hashScreenshotIdentifier(result.identifier);
|
|
18
|
-
if (retryDiffById.has(hash)) {
|
|
19
|
-
throw new Error(`Received two screenshots for the same identifier '${hash}'. Screenshots should be unique.`);
|
|
20
|
-
}
|
|
21
|
-
retryDiffById.set(hash, result);
|
|
22
|
-
});
|
|
23
|
-
const newScreenshotDiffResults = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(currentResult).map((currentDiff) => {
|
|
24
|
-
const hash = hashScreenshotIdentifier(currentDiff.identifier);
|
|
25
|
-
const diffWhenRetrying = retryDiffById.get(hash);
|
|
26
|
-
// diffWhenRetrying is null in the case that there is a base screenshot for the base replay,
|
|
27
|
-
// but the first replay on head did not generate a head screenshot (got 'missing-head'),
|
|
28
|
-
// and the replay of that did not generate a head screenshot either (if the first replay on head
|
|
29
|
-
// did generate a screenshot, then diffWhenRetrying.outcome would be 'missing-head' instead)
|
|
30
|
-
if (diffWhenRetrying == null) {
|
|
31
|
-
const diffToBaseScreenshotOutcome = currentDiff.outcome === "flake"
|
|
32
|
-
? currentDiff.diffToBaseScreenshot.outcome
|
|
33
|
-
: currentDiff.outcome;
|
|
34
|
-
if (diffToBaseScreenshotOutcome !== "missing-head") {
|
|
35
|
-
throw new Error(`Expected to find a screenshot comparison for ${hash}, but none was found. The screenshot must
|
|
36
|
-
have existed in the orginal replay, since the orginal comparison outcome was not '${diffToBaseScreenshotOutcome}'.`);
|
|
37
|
-
}
|
|
38
|
-
if (currentDiff.outcome === "flake") {
|
|
39
|
-
// Original comparison had missing-head, so we re-ran to check if it was flakey. In this case
|
|
40
|
-
// we got the same result.
|
|
41
|
-
return {
|
|
42
|
-
...currentDiff,
|
|
43
|
-
diffsToHeadScreenshotOnRetries: [
|
|
44
|
-
...currentDiff.diffsToHeadScreenshotOnRetries,
|
|
45
|
-
{ outcome: "missing-base-and-head" },
|
|
46
|
-
],
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
return currentDiff; // no difference, both screenshots were missing
|
|
50
|
-
}
|
|
51
|
-
if (diffWhenRetrying.outcome === "flake") {
|
|
52
|
-
throw new Error("Expected diffs when retrying and comparing to the original head screenshot to be first-try diffs, but got a flake.");
|
|
53
|
-
}
|
|
54
|
-
if (currentDiff.outcome === "flake") {
|
|
55
|
-
return {
|
|
56
|
-
...currentDiff,
|
|
57
|
-
diffsToHeadScreenshotOnRetries: [
|
|
58
|
-
...currentDiff.diffsToHeadScreenshotOnRetries,
|
|
59
|
-
withoutIdentifiers(diffWhenRetrying),
|
|
60
|
-
],
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
else if ((diffWhenRetrying === null || diffWhenRetrying === void 0 ? void 0 : diffWhenRetrying.outcome) === "no-diff") {
|
|
64
|
-
return currentDiff;
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
return {
|
|
68
|
-
baseReplayId: currentDiff.baseReplayId,
|
|
69
|
-
identifier: currentDiff.identifier,
|
|
70
|
-
outcome: "flake",
|
|
71
|
-
diffToBaseScreenshot: withoutIdentifiers(currentDiff),
|
|
72
|
-
diffsToHeadScreenshotOnRetries: [withoutIdentifiers(diffWhenRetrying)],
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
});
|
|
76
|
-
return {
|
|
77
|
-
...currentResult,
|
|
78
|
-
result: (0, exports.testRunOutcomeFromDiffResults)(currentResult.result, newScreenshotDiffResults),
|
|
79
|
-
screenshotDiffResultsByBaseReplayId: (0, screenshot_diff_results_utils_1.groupScreenshotDiffResults)(newScreenshotDiffResults),
|
|
80
|
-
};
|
|
81
|
-
};
|
|
82
|
-
exports.mergeResults = mergeResults;
|
|
83
|
-
const withoutIdentifiers = ({
|
|
84
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
85
|
-
identifier: _identifier,
|
|
86
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
87
|
-
baseReplayId: _baseReplayId, ...rest }) => rest;
|
|
88
|
-
const hashScreenshotIdentifier = (identifier) => {
|
|
89
|
-
if (identifier.type === "end-state") {
|
|
90
|
-
return "end-state";
|
|
91
|
-
}
|
|
92
|
-
else if (identifier.type === "after-event") {
|
|
93
|
-
return `after-event-${identifier.eventNumber}`;
|
|
94
|
-
}
|
|
95
|
-
else {
|
|
96
|
-
unknownScreenshotIdentifierType(identifier);
|
|
97
|
-
// The identifier is probably from a newer version of the bundle script
|
|
98
|
-
// and we're on an old version of the CLI. Our best bet is to stringify it
|
|
99
|
-
// and use that as a hash.
|
|
100
|
-
return (0, fast_json_stable_stringify_1.default)(identifier);
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
const unknownScreenshotIdentifierType = (identifier) => {
|
|
104
|
-
utils_1.logger.error(`Unknown type of screenshot identifier: ${JSON.stringify(identifier)}`);
|
|
105
|
-
};
|
|
106
|
-
const testRunOutcomeFromDiffResults = (currentResult, newScreenshotDiffResults) => {
|
|
107
|
-
// If a test run is already flakey, we don't want to overwrite that with a 'fail' result.
|
|
108
|
-
if (currentResult === "flake") {
|
|
109
|
-
return "flake";
|
|
110
|
-
}
|
|
111
|
-
const hasOnlyFlakes = newScreenshotDiffResults.every(({ outcome }) => outcome === "flake" || outcome === "no-diff");
|
|
112
|
-
if (hasOnlyFlakes) {
|
|
113
|
-
return "flake";
|
|
114
|
-
}
|
|
115
|
-
// If we've had a test run has already failed, we don't want to overwrite that with a 'pass' result.
|
|
116
|
-
// Otherwise, if there are any notable differences, we want to fail the test run.
|
|
117
|
-
return currentResult === "fail" ||
|
|
118
|
-
(0, has_notable_differences_1.hasNotableDifferences)(newScreenshotDiffResults)
|
|
119
|
-
? "fail"
|
|
120
|
-
: "pass";
|
|
121
|
-
};
|
|
122
|
-
exports.testRunOutcomeFromDiffResults = testRunOutcomeFromDiffResults;
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import log from "loglevel";
|
|
2
|
-
import { DetailedTestCaseResult } from "../config/config.types";
|
|
3
|
-
import { ParallelTestsReplayOptions } from "./parallel-replay.types";
|
|
4
|
-
export interface InitMessage {
|
|
5
|
-
kind: "init";
|
|
6
|
-
data: {
|
|
7
|
-
logLevel: log.LogLevel[keyof log.LogLevel];
|
|
8
|
-
dataDir: string;
|
|
9
|
-
replayOptions: ParallelTestsReplayOptions;
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
export interface ResultMessage {
|
|
13
|
-
kind: "result";
|
|
14
|
-
data: {
|
|
15
|
-
result: DetailedTestCaseResult;
|
|
16
|
-
};
|
|
17
|
-
}
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
import { DetailedTestCaseResult } from "../config/config.types";
|
|
2
|
-
import { ParallelTestsReplayOptions } from "./parallel-replay.types";
|
|
3
|
-
export declare const handleReplay: ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, baseTestRunId, }: ParallelTestsReplayOptions) => Promise<DetailedTestCaseResult>;
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.handleReplay = void 0;
|
|
4
|
-
const replay_command_1 = require("../commands/replay/replay.command");
|
|
5
|
-
const has_notable_differences_1 = require("../commands/screenshot-diff/utils/has-notable-differences");
|
|
6
|
-
const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, baseTestRunId, }) => {
|
|
7
|
-
var _a, _b;
|
|
8
|
-
const { replay, screenshotDiffResultsByBaseReplayId } = await (0, replay_command_1.replayCommandHandler)({
|
|
9
|
-
replayTarget,
|
|
10
|
-
executionOptions: applyTestCaseExecutionOptionOverrides(executionOptions, (_a = testCase.options) !== null && _a !== void 0 ? _a : {}),
|
|
11
|
-
screenshottingOptions: applyTestCaseScreenshottingOptionsOverrides(screenshottingOptions, (_b = testCase.options) !== null && _b !== void 0 ? _b : {}),
|
|
12
|
-
apiToken,
|
|
13
|
-
commitSha,
|
|
14
|
-
sessionId: testCase.sessionId,
|
|
15
|
-
baseTestRunId,
|
|
16
|
-
cookiesFile: null,
|
|
17
|
-
generatedBy,
|
|
18
|
-
testRunId,
|
|
19
|
-
replayEventsDependencies,
|
|
20
|
-
suppressScreenshotDiffLogging,
|
|
21
|
-
debugger: false,
|
|
22
|
-
});
|
|
23
|
-
const result = (0, has_notable_differences_1.hasNotableDifferences)(Object.values(screenshotDiffResultsByBaseReplayId).flat())
|
|
24
|
-
? "fail"
|
|
25
|
-
: "pass";
|
|
26
|
-
return {
|
|
27
|
-
...testCase,
|
|
28
|
-
headReplayId: replay.id,
|
|
29
|
-
result,
|
|
30
|
-
screenshotDiffResultsByBaseReplayId,
|
|
31
|
-
};
|
|
32
|
-
};
|
|
33
|
-
exports.handleReplay = handleReplay;
|
|
34
|
-
const applyTestCaseExecutionOptionOverrides = (executionOptionsFromCliFlags, overridesFromTestCase) => {
|
|
35
|
-
var _a;
|
|
36
|
-
// Options specified in the test case override those passed as CLI flags
|
|
37
|
-
// (CLI flags set the defaults)
|
|
38
|
-
return {
|
|
39
|
-
...executionOptionsFromCliFlags,
|
|
40
|
-
moveBeforeClick: (_a = overridesFromTestCase.moveBeforeClick) !== null && _a !== void 0 ? _a : executionOptionsFromCliFlags.moveBeforeClick,
|
|
41
|
-
};
|
|
42
|
-
};
|
|
43
|
-
const applyTestCaseScreenshottingOptionsOverrides = (screenshottingOptionsFromCliFlags, overridesFromTestCase) => {
|
|
44
|
-
var _a, _b, _c;
|
|
45
|
-
// Options specified in the test case override those passed as CLI flags
|
|
46
|
-
// (CLI flags set the defaults)
|
|
47
|
-
const diffOptions = {
|
|
48
|
-
diffThreshold: (_a = overridesFromTestCase.diffThreshold) !== null && _a !== void 0 ? _a : screenshottingOptionsFromCliFlags.diffOptions.diffThreshold,
|
|
49
|
-
diffPixelThreshold: (_b = overridesFromTestCase.diffPixelThreshold) !== null && _b !== void 0 ? _b : screenshottingOptionsFromCliFlags.diffOptions.diffPixelThreshold,
|
|
50
|
-
};
|
|
51
|
-
return {
|
|
52
|
-
enabled: true,
|
|
53
|
-
screenshotSelector: (_c = overridesFromTestCase.screenshotSelector) !== null && _c !== void 0 ? _c : screenshottingOptionsFromCliFlags.screenshotSelector,
|
|
54
|
-
diffOptions,
|
|
55
|
-
storyboardOptions: screenshottingOptionsFromCliFlags.storyboardOptions,
|
|
56
|
-
};
|
|
57
|
-
};
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { TestCase } from "@alwaysmeticulous/api";
|
|
2
|
-
import { GeneratedBy, ReplayEventsDependencies, ReplayExecutionOptions, ReplayTarget } from "@alwaysmeticulous/common";
|
|
3
|
-
import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
|
|
4
|
-
export interface ParallelTestsReplayOptions {
|
|
5
|
-
replayTarget: ReplayTarget;
|
|
6
|
-
executionOptions: ReplayExecutionOptions;
|
|
7
|
-
screenshottingOptions: ScreenshotAssertionsEnabledOptions;
|
|
8
|
-
testCase: TestCase;
|
|
9
|
-
apiToken: string | null;
|
|
10
|
-
commitSha: string;
|
|
11
|
-
generatedBy: GeneratedBy;
|
|
12
|
-
testRunId: string | null;
|
|
13
|
-
baseTestRunId: string | null;
|
|
14
|
-
replayEventsDependencies: ReplayEventsDependencies;
|
|
15
|
-
suppressScreenshotDiffLogging: boolean;
|
|
16
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { TestCase } from "@alwaysmeticulous/api";
|
|
2
|
-
import { DetailedTestCaseResult } from "../config/config.types";
|
|
3
|
-
import { TestRunProgress } from "./run-all-tests.types";
|
|
4
|
-
export interface RunAllTestsInParallelOptions {
|
|
5
|
-
testsToRun: TestCase[];
|
|
6
|
-
parallelTasks: number | null;
|
|
7
|
-
maxRetriesOnFailure: number;
|
|
8
|
-
rerunTestsNTimes: number;
|
|
9
|
-
executeTest: (testCase: TestCase, isRetry: boolean) => Promise<DetailedTestCaseResult>;
|
|
10
|
-
onTestFinished?: (progress: TestRunProgress, resultsSoFar: DetailedTestCaseResult[]) => Promise<void>;
|
|
11
|
-
onTestFailedToRun?: (progress: TestRunProgress) => Promise<void>;
|
|
12
|
-
}
|
|
13
|
-
/** Handler for running Meticulous tests in parallel using child processes */
|
|
14
|
-
export declare const runAllTestsInParallel: (options: RunAllTestsInParallelOptions) => Promise<DetailedTestCaseResult[]>;
|