@alwaysmeticulous/cli 2.40.0 → 2.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/api/test-run.api.d.ts +3 -31
  2. package/dist/api/test-run.api.js +3 -96
  3. package/dist/commands/download-replay/download-replay.command.js +4 -4
  4. package/dist/commands/download-session/download-session.command.js +4 -4
  5. package/dist/commands/record/record.command.js +8 -11
  6. package/dist/commands/replay/replay.command.d.ts +11 -20
  7. package/dist/commands/replay/replay.command.js +20 -235
  8. package/dist/commands/run-all-tests/run-all-tests.command.js +5 -5
  9. package/dist/commands/show-project/show-project.command.js +2 -3
  10. package/dist/index.d.ts +0 -7
  11. package/dist/index.js +1 -13
  12. package/dist/main.js +5 -11
  13. package/dist/utils/sentry.utils.d.ts +0 -4
  14. package/dist/utils/sentry.utils.js +3 -23
  15. package/package.json +14 -32
  16. package/dist/api/client.d.ts +0 -5
  17. package/dist/api/client.js +0 -32
  18. package/dist/api/download.d.ts +0 -1
  19. package/dist/api/download.js +0 -21
  20. package/dist/api/project.api.d.ts +0 -3
  21. package/dist/api/project.api.js +0 -22
  22. package/dist/api/replay-diff.api.d.ts +0 -13
  23. package/dist/api/replay-diff.api.js +0 -32
  24. package/dist/api/replay.api.d.ts +0 -36
  25. package/dist/api/replay.api.js +0 -89
  26. package/dist/api/session.api.d.ts +0 -6
  27. package/dist/api/session.api.js +0 -45
  28. package/dist/api/types.d.ts +0 -39
  29. package/dist/api/types.js +0 -2
  30. package/dist/api/upload.d.ts +0 -1
  31. package/dist/api/upload.js +0 -18
  32. package/dist/archive/archive.d.ts +0 -4
  33. package/dist/archive/archive.js +0 -64
  34. package/dist/command-utils/common-types.d.ts +0 -20
  35. package/dist/command-utils/common-types.js +0 -2
  36. package/dist/commands/bootstrap/bootstrap.command.d.ts +0 -2
  37. package/dist/commands/bootstrap/bootstrap.command.js +0 -33
  38. package/dist/commands/create-test/create-test.command.d.ts +0 -90
  39. package/dist/commands/create-test/create-test.command.js +0 -165
  40. package/dist/commands/replay/utils/compute-diff.d.ts +0 -13
  41. package/dist/commands/replay/utils/compute-diff.js +0 -95
  42. package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.d.ts +0 -1
  43. package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.js +0 -27
  44. package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts +0 -53
  45. package/dist/commands/screenshot-diff/screenshot-diff.command.js +0 -182
  46. package/dist/commands/screenshot-diff/utils/__tests__/get-screenshot-identifier.spec.d.ts +0 -1
  47. package/dist/commands/screenshot-diff/utils/__tests__/get-screenshot-identifier.spec.js +0 -25
  48. package/dist/commands/screenshot-diff/utils/get-screenshot-filename.d.ts +0 -2
  49. package/dist/commands/screenshot-diff/utils/get-screenshot-filename.js +0 -22
  50. package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.d.ts +0 -2
  51. package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.js +0 -39
  52. package/dist/commands/screenshot-diff/utils/has-notable-differences.d.ts +0 -2
  53. package/dist/commands/screenshot-diff/utils/has-notable-differences.js +0 -10
  54. package/dist/commands/serve/serve.command.d.ts +0 -10
  55. package/dist/commands/serve/serve.command.js +0 -31
  56. package/dist/commands/update-tests/update-tests.command.d.ts +0 -21
  57. package/dist/commands/update-tests/update-tests.command.js +0 -96
  58. package/dist/config/config.d.ts +0 -3
  59. package/dist/config/config.js +0 -69
  60. package/dist/config/config.types.d.ts +0 -16
  61. package/dist/config/config.types.js +0 -2
  62. package/dist/config/snippets.d.ts +0 -1
  63. package/dist/config/snippets.js +0 -18
  64. package/dist/errors/config.d.ts +0 -3
  65. package/dist/errors/config.js +0 -10
  66. package/dist/image/diff.utils.d.ts +0 -19
  67. package/dist/image/diff.utils.js +0 -25
  68. package/dist/image/io.utils.d.ts +0 -3
  69. package/dist/image/io.utils.js +0 -25
  70. package/dist/local-data/local-data.utils.d.ts +0 -20
  71. package/dist/local-data/local-data.utils.js +0 -92
  72. package/dist/local-data/replay-assets.d.ts +0 -3
  73. package/dist/local-data/replay-assets.js +0 -94
  74. package/dist/local-data/replays.d.ts +0 -18
  75. package/dist/local-data/replays.js +0 -87
  76. package/dist/local-data/screenshot-diffs.d.ts +0 -7
  77. package/dist/local-data/screenshot-diffs.js +0 -20
  78. package/dist/local-data/serve-assets-from-simulation.d.ts +0 -5
  79. package/dist/local-data/serve-assets-from-simulation.js +0 -52
  80. package/dist/local-data/sessions.d.ts +0 -10
  81. package/dist/local-data/sessions.js +0 -41
  82. package/dist/parallel-tests/__tests__/merge-test-results.spec.d.ts +0 -1
  83. package/dist/parallel-tests/__tests__/merge-test-results.spec.js +0 -104
  84. package/dist/parallel-tests/__tests__/mock-test-results.d.ts +0 -17
  85. package/dist/parallel-tests/__tests__/mock-test-results.js +0 -106
  86. package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.d.ts +0 -1
  87. package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.js +0 -205
  88. package/dist/parallel-tests/execute-test-in-child-process.d.ts +0 -3
  89. package/dist/parallel-tests/execute-test-in-child-process.js +0 -44
  90. package/dist/parallel-tests/merge-test-results.d.ts +0 -12
  91. package/dist/parallel-tests/merge-test-results.js +0 -106
  92. package/dist/parallel-tests/messages.types.d.ts +0 -17
  93. package/dist/parallel-tests/messages.types.js +0 -2
  94. package/dist/parallel-tests/parallel-replay.handler.d.ts +0 -3
  95. package/dist/parallel-tests/parallel-replay.handler.js +0 -56
  96. package/dist/parallel-tests/parallel-replay.types.d.ts +0 -16
  97. package/dist/parallel-tests/parallel-replay.types.js +0 -2
  98. package/dist/parallel-tests/parallel-tests.handler.d.ts +0 -14
  99. package/dist/parallel-tests/parallel-tests.handler.js +0 -194
  100. package/dist/parallel-tests/run-all-tests.d.ts +0 -73
  101. package/dist/parallel-tests/run-all-tests.js +0 -237
  102. package/dist/parallel-tests/run-all-tests.types.d.ts +0 -6
  103. package/dist/parallel-tests/run-all-tests.types.js +0 -2
  104. package/dist/parallel-tests/screenshot-diff-results.utils.d.ts +0 -7
  105. package/dist/parallel-tests/screenshot-diff-results.utils.js +0 -20
  106. package/dist/parallel-tests/task.handler.d.ts +0 -1
  107. package/dist/parallel-tests/task.handler.js +0 -86
  108. package/dist/utils/api-token.utils.d.ts +0 -1
  109. package/dist/utils/api-token.utils.js +0 -31
  110. package/dist/utils/commit-sha.utils.d.ts +0 -1
  111. package/dist/utils/commit-sha.utils.js +0 -42
  112. package/dist/utils/config.utils.d.ts +0 -7
  113. package/dist/utils/config.utils.js +0 -33
  114. package/dist/utils/github-summary.utils.d.ts +0 -5
  115. package/dist/utils/github-summary.utils.js +0 -38
  116. package/dist/utils/run-all-tests.utils.d.ts +0 -16
  117. package/dist/utils/run-all-tests.utils.js +0 -32
  118. package/dist/utils/test-run-environment.utils.d.ts +0 -2
  119. package/dist/utils/test-run-environment.utils.js +0 -25
  120. package/dist/utils/version.utils.d.ts +0 -1
  121. package/dist/utils/version.utils.js +0 -17
@@ -1,194 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.runAllTestsInParallel = void 0;
7
- const os_1 = require("os");
8
- const common_1 = require("@alwaysmeticulous/common");
9
- const loglevel_1 = __importDefault(require("loglevel"));
10
- const merge_test_results_1 = require("./merge-test-results");
11
- const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
12
- /** Handler for running Meticulous tests in parallel using child processes */
13
- const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, executeTest, onTestFinished, onTestFailedToRun, }) => {
14
- if (maxRetriesOnFailure && rerunTestsNTimes) {
15
- throw new Error("maxRetriesOnFailure and rerunTestsNTimes are mutually exclusive.");
16
- }
17
- const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
18
- let nextId = 0;
19
- let queue = testsToRun.map((test) => ({
20
- ...test,
21
- id: ++nextId,
22
- }));
23
- /**
24
- * The current results, which may still be being updated if we're re-running a test
25
- * to check for flakes.
26
- */
27
- const resultsByTestId = new Map();
28
- /**
29
- * Results that have been fully checked for flakes. At most one per test case.
30
- */
31
- const finalResults = [];
32
- let progress = {
33
- runningTestCases: queue.length,
34
- failedTestCases: 0,
35
- flakedTestCases: 0,
36
- passedTestCases: 0,
37
- };
38
- const allTasksDone = (0, common_1.defer)();
39
- let inProgress = 0;
40
- // Average run time when testing on a 16 core machine: 16 parallel tasks @ 56s, 32 parallel tasks @ 52.5s
41
- const maxTasks = parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : Math.max((0, os_1.cpus)().length, 1) * 2;
42
- logger.debug(`Running with ${maxTasks} maximum tasks in parallel`);
43
- // Starts running a test case in a child process
44
- const startTask = (rerunnableTestCase) => {
45
- const { id, ...testCase } = rerunnableTestCase;
46
- const isRetry = resultsByTestId.has(id);
47
- // Handle task completion
48
- executeTest(testCase, isRetry)
49
- .catch(() => null)
50
- .then(async (result) => {
51
- var _a;
52
- const resultsForTestCase = resultsByTestId.get(id);
53
- // Re-run the test if needed, comparing to the base replay.
54
- if (!isRetry) {
55
- queue.push(...Array.from(new Array(rerunTestsNTimes)).map(() => ({
56
- ...testCase,
57
- id,
58
- baseReplayId: result === null || result === void 0 ? void 0 : result.headReplayId,
59
- })));
60
- }
61
- if (resultsForTestCase != null && result != null) {
62
- logRetrySummary(testName({ id, ...testCase }), result, resultsForTestCase.currentResult);
63
- }
64
- if (resultsForTestCase != null &&
65
- resultsForTestCase.currentResult.result === "flake") {
66
- // This test has already been declared as flakey, and we can ignore this result. This result
67
- // was from an already executing test that we weren't able to cancel.
68
- return;
69
- }
70
- if ((result === null || result === void 0 ? void 0 : result.result) === "fail" && resultsForTestCase == null) {
71
- // Let's auto-retry to see if this failure persists
72
- queue.push(...Array.from(new Array(maxRetriesOnFailure)).map(() => ({
73
- ...testCase,
74
- id,
75
- baseReplayId: result.headReplayId,
76
- })));
77
- }
78
- if (result == null && (resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) == null) {
79
- // This means our original head replay failed fatally (not just a failed diff, but failed to even run)
80
- progress = updateProgress(progress, "fail");
81
- await (onTestFailedToRun === null || onTestFailedToRun === void 0 ? void 0 : onTestFailedToRun(progress));
82
- return;
83
- }
84
- const mergedResult = getNewMergedResult((_a = resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) !== null && _a !== void 0 ? _a : null, result);
85
- const numberOfRetriesExecuted = resultsForTestCase == null
86
- ? 0
87
- : resultsForTestCase.numberOfRetriesExecuted + 1;
88
- resultsByTestId.set(id, {
89
- currentResult: mergedResult,
90
- numberOfRetriesExecuted,
91
- });
92
- // Our work is done for this test case if the first result was a pass,
93
- // we've performed all the retries, or one of the retries already proved
94
- // the result as flakey
95
- let isFinalResult;
96
- if (rerunTestsNTimes > 0) {
97
- isFinalResult = numberOfRetriesExecuted >= rerunTestsNTimes;
98
- }
99
- else {
100
- isFinalResult =
101
- mergedResult.result !== "fail" ||
102
- numberOfRetriesExecuted >= maxRetriesOnFailure;
103
- }
104
- if (isFinalResult) {
105
- // Cancel any replays that are still scheduled
106
- queue = queue.filter((otherReplay) => otherReplay.id !== id);
107
- finalResults.push(mergedResult);
108
- progress = updateProgress(progress, mergedResult.result);
109
- await (onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, finalResults));
110
- }
111
- })
112
- .catch((err) => {
113
- logger.error(`Error processing result of completed task for test '${testName({
114
- id,
115
- ...testCase,
116
- })}'`, err);
117
- })
118
- .finally(() => {
119
- // We only decrement inProgress at the very end,
120
- // otherwise another promise may call allTasksDone.resolve() while
121
- // we're still saving results
122
- --inProgress;
123
- if (queue.length === 0 && inProgress === 0) {
124
- allTasksDone.resolve();
125
- }
126
- process.nextTick(checkNextTask);
127
- });
128
- };
129
- // Checks if we can start a new child process
130
- const checkNextTask = () => {
131
- if (inProgress >= maxTasks) {
132
- return;
133
- }
134
- const testCase = queue.shift();
135
- if (!testCase) {
136
- return;
137
- }
138
- ++inProgress;
139
- if (resultsByTestId.has(testCase.id)) {
140
- if (maxRetriesOnFailure > 0) {
141
- logger.info(`Test ${testName(testCase)} failed. Retrying to check for flakes...`);
142
- }
143
- else if (rerunTestsNTimes > 0) {
144
- logger.info(`Re-running ${testName(testCase)} to check for flakes...`);
145
- }
146
- }
147
- startTask(testCase);
148
- process.nextTick(checkNextTask);
149
- };
150
- process.nextTick(checkNextTask);
151
- await allTasksDone.promise;
152
- return finalResults;
153
- };
154
- exports.runAllTestsInParallel = runAllTestsInParallel;
155
- const logRetrySummary = (nameOfTest, retryResult, resultSoFar) => {
156
- const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
157
- if (retryResult.result === "pass") {
158
- logger.info(`Retried taking screenshots for${resultSoFar.result == "fail" ? " failed" : ""} test ${nameOfTest}, but got the same results`);
159
- }
160
- else {
161
- const numDifferingScreenshots = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(retryResult).filter((result) => result.outcome !== "no-diff").length;
162
- logger.info(`Retried taking screenshots for failed test ${nameOfTest}, and ${numDifferingScreenshots} screenshots came out different. Results for these screenshots are assumed to be flakes, and so will be ignored.`);
163
- }
164
- };
165
- const testName = (testCase) => testCase.title != null ? `'${testCase.title}'` : `#${testCase.id + 1}`;
166
- const getNewMergedResult = (currentMergedResult, newResult) => {
167
- // If currentMergedResult is null then this is our first try, our original head replay
168
- if (currentMergedResult == null) {
169
- if (newResult == null) {
170
- throw new Error("Expected either newResult to be non-null or currentMergedResult to be non-null, but both were null. This case should be handled before getNewMergedResult is called");
171
- }
172
- // In this case the newResult is our first head replay, our first result,
173
- // so lets initialize the mergedResult to this
174
- return newResult;
175
- }
176
- // If currentMergedResult is not null then newResult is a retry, containing comparison screenshots to our original head replay
177
- if (newResult == null) {
178
- // In this case newResult is a retry of the head replay, but it failed fatally (not just a failed diff, but failed to even run)
179
- // So we just ignore this retry
180
- return currentMergedResult;
181
- }
182
- return (0, merge_test_results_1.mergeResults)({
183
- currentResult: currentMergedResult,
184
- comparisonToHeadReplay: newResult,
185
- });
186
- };
187
- const updateProgress = (progress, newResult) => {
188
- return {
189
- runningTestCases: progress.runningTestCases - 1,
190
- failedTestCases: progress.failedTestCases + (newResult === "fail" ? 1 : 0),
191
- flakedTestCases: progress.flakedTestCases + (newResult === "flake" ? 1 : 0),
192
- passedTestCases: progress.passedTestCases + (newResult === "pass" ? 1 : 0),
193
- };
194
- };
@@ -1,73 +0,0 @@
1
- import { TestRunEnvironment } from "@alwaysmeticulous/api";
2
- import { ReplayExecutionOptions } from "@alwaysmeticulous/common";
3
- import { TestRun } from "../api/types";
4
- import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
5
- import { DetailedTestCaseResult, TestCaseResult } from "../config/config.types";
6
- import { TestRunProgress } from "./run-all-tests.types";
7
- export type RunAllTestsTestRun = Pick<TestRun, "id" | "url" | "status" | "project"> & {
8
- progress: TestRunProgress;
9
- url: string;
10
- };
11
- export interface Options {
12
- testsFile: string | null;
13
- executionOptions: ReplayExecutionOptions;
14
- screenshottingOptions: ScreenshotAssertionsEnabledOptions;
15
- apiToken: string | null;
16
- commitSha: string;
17
- /**
18
- * The base commit to compare test results against for test cases that don't have a baseReplayId specified.
19
- */
20
- baseCommitSha: string | null;
21
- appUrl: string | null;
22
- /**
23
- * If null runs in parralel with a sensible number of parrelel tasks for the given machine.
24
- *
25
- * Set to 1 to disable parralelism.
26
- */
27
- parallelTasks: number | null;
28
- /**
29
- * If set to a value greater than 1 then will re-run any replays that give a screenshot diff
30
- * and mark them as a flake if the screenshot generated on one of the retryed replays differs from that
31
- * in the first replay.
32
- */
33
- maxRetriesOnFailure: number;
34
- /**
35
- * If set to a value greater than 0 then will re-run all replays the specified number of times
36
- * and mark them as a flake if the screenshot generated on one of the retryed replays differs from that
37
- * in the first replay.
38
- *
39
- * This is useful for checking flake rates.
40
- *
41
- * This option is mutually exclusive with maxRetriesOnFailure.
42
- */
43
- rerunTestsNTimes: number;
44
- githubSummary: boolean;
45
- /**
46
- * If provided it will incorportate the cachedTestRunResults in any calls to store
47
- * test run results in the BE, but won't include the cachedTestRunResults in the returned
48
- * RunAllTestsResult.
49
- */
50
- cachedTestRunResults?: TestCaseResult[];
51
- /**
52
- * Captured environment for this run
53
- */
54
- environment?: TestRunEnvironment;
55
- baseTestRunId: string | null;
56
- onTestRunCreated?: (testRun: RunAllTestsTestRun & {
57
- status: "Running";
58
- }) => void;
59
- onTestFinished?: (testRun: RunAllTestsTestRun & {
60
- status: "Running";
61
- }) => void;
62
- }
63
- export interface RunAllTestsResult {
64
- testRun: RunAllTestsTestRun & {
65
- status: "Success" | "Failure";
66
- };
67
- testCaseResults: DetailedTestCaseResult[];
68
- }
69
- /**
70
- * Runs all the test cases in the provided file.
71
- * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
72
- */
73
- export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;
@@ -1,237 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.runAllTests = void 0;
7
- const common_1 = require("@alwaysmeticulous/common");
8
- const loglevel_1 = __importDefault(require("loglevel"));
9
- const client_1 = require("../api/client");
10
- const project_api_1 = require("../api/project.api");
11
- const replay_diff_api_1 = require("../api/replay-diff.api");
12
- const test_run_api_1 = require("../api/test-run.api");
13
- const config_1 = require("../config/config");
14
- const replay_assets_1 = require("../local-data/replay-assets");
15
- const config_utils_1 = require("../utils/config.utils");
16
- const github_summary_utils_1 = require("../utils/github-summary.utils");
17
- const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
18
- const test_run_environment_utils_1 = require("../utils/test-run-environment.utils");
19
- const version_utils_1 = require("../utils/version.utils");
20
- const execute_test_in_child_process_1 = require("./execute-test-in-child-process");
21
- const parallel_tests_handler_1 = require("./parallel-tests.handler");
22
- /**
23
- * Runs all the test cases in the provided file.
24
- * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
25
- */
26
- const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
27
- const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
28
- const client = (0, client_1.createClient)({ apiToken });
29
- const project = await (0, project_api_1.getProject)(client);
30
- if (!project) {
31
- logger.error("Could not retrieve project data. Is the API token correct?");
32
- process.exit(1);
33
- }
34
- const cachedTestRunResults = cachedTestRunResults_ !== null && cachedTestRunResults_ !== void 0 ? cachedTestRunResults_ : [];
35
- const config = await (0, config_1.readConfig)(testsFile || undefined);
36
- const allTestCases = (0, run_all_tests_utils_1.mergeTestCases)(project.configurationData.testCases, config.testCases);
37
- if (!allTestCases.length) {
38
- throw new Error("Error! No test case defined");
39
- }
40
- // Only run the uncached test cases
41
- const testCases = allTestCases.filter(({ sessionId, baseTestRunId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
42
- cached.baseTestRunId === baseTestRunId &&
43
- cached.title === title));
44
- const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
45
- const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
46
- const testRun = await (0, test_run_api_1.createTestRun)({
47
- client,
48
- commitSha,
49
- meticulousSha,
50
- configData: {
51
- ...config,
52
- testCases: allTestCases,
53
- arguments: {
54
- executionOptions,
55
- screenshottingOptions,
56
- commitSha,
57
- baseCommitSha,
58
- appUrl,
59
- parallelTasks,
60
- githubSummary,
61
- },
62
- environment: (0, test_run_environment_utils_1.getEnvironment)(environment),
63
- },
64
- });
65
- const testRunUrl = (0, test_run_api_1.getTestRunUrl)(testRun);
66
- onTestRunCreated === null || onTestRunCreated === void 0 ? void 0 : onTestRunCreated({
67
- id: testRun.id,
68
- url: testRunUrl,
69
- project: testRun.project,
70
- status: "Running",
71
- progress: {
72
- failedTestCases: 0,
73
- flakedTestCases: 0,
74
- passedTestCases: cachedTestRunResults.length,
75
- runningTestCases: testCases.length,
76
- },
77
- });
78
- logger.info("");
79
- logger.info(`Test run URL: ${testRunUrl}`);
80
- logger.info("");
81
- const testsToRun = await getTestCasesWithBaseTestRunId({
82
- baseCommitSha,
83
- baseTestRunId: baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : null,
84
- client,
85
- logger,
86
- testCases,
87
- });
88
- const storeTestRunResults = async (status, resultsSoFar) => {
89
- const resultsToSendToBE = [
90
- ...cachedTestRunResults,
91
- ...resultsSoFar.map(
92
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
93
- ({ screenshotDiffResultsByBaseReplayId, ...result }) => result),
94
- ];
95
- try {
96
- await (0, test_run_api_1.putTestRunResults)({
97
- client,
98
- testRunId: testRun.id,
99
- status,
100
- resultData: {
101
- results: resultsToSendToBE,
102
- },
103
- });
104
- }
105
- catch (error) {
106
- logger.error(`Error while pushing partial results: ${error}`);
107
- }
108
- };
109
- const onProgressUpdated = async (progress) => {
110
- onTestFinished_ === null || onTestFinished_ === void 0 ? void 0 : onTestFinished_({
111
- id: testRun.id,
112
- url: testRunUrl,
113
- project: testRun.project,
114
- status: "Running",
115
- progress: {
116
- ...progress,
117
- passedTestCases: progress.passedTestCases + cachedTestRunResults.length,
118
- },
119
- });
120
- };
121
- const onTestFinished = async (progress, resultsSoFar) => {
122
- onProgressUpdated(progress);
123
- const newResult = resultsSoFar.at(-1);
124
- if (newResult != null) {
125
- for (const [baseReplayId, screenshotDiffResults] of Object.entries(newResult.screenshotDiffResultsByBaseReplayId)) {
126
- await (0, replay_diff_api_1.createReplayDiff)({
127
- client,
128
- headReplayId: newResult.headReplayId,
129
- baseReplayId: baseReplayId,
130
- testRunId: testRun.id,
131
- data: {
132
- screenshotAssertionsOptions: screenshottingOptions,
133
- screenshotDiffResults,
134
- },
135
- });
136
- }
137
- }
138
- await storeTestRunResults("Running", resultsSoFar);
139
- };
140
- const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
141
- testsToRun,
142
- parallelTasks,
143
- maxRetriesOnFailure,
144
- rerunTestsNTimes,
145
- executeTest: (testCase, isRetry) => {
146
- var _a;
147
- const initMessage = {
148
- kind: "init",
149
- data: {
150
- logLevel: logger.getLevel(),
151
- dataDir: (0, common_1.getMeticulousLocalDataDir)(),
152
- replayOptions: {
153
- apiToken,
154
- commitSha,
155
- testCase,
156
- replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
157
- appUrl,
158
- testCase,
159
- }),
160
- executionOptions,
161
- screenshottingOptions,
162
- generatedBy: { type: "testRun", runId: testRun.id },
163
- testRunId: testRun.id,
164
- baseTestRunId: (_a = testCase.baseTestRunId) !== null && _a !== void 0 ? _a : null,
165
- replayEventsDependencies,
166
- suppressScreenshotDiffLogging: isRetry,
167
- },
168
- },
169
- };
170
- return (0, execute_test_in_child_process_1.executeTestInChildProcess)(initMessage);
171
- },
172
- onTestFinished,
173
- onTestFailedToRun: onProgressUpdated,
174
- });
175
- const sortedResults = (0, run_all_tests_utils_1.sortResults)({
176
- results: results,
177
- testCases: config.testCases || [],
178
- });
179
- const runAllFailure = sortedResults.find(({ result }) => result === "fail");
180
- const overallStatus = runAllFailure ? "Failure" : "Success";
181
- await storeTestRunResults(overallStatus, sortedResults);
182
- logger.info("");
183
- logger.info("Results");
184
- logger.info("=======");
185
- logger.info(`URL: ${testRunUrl}`);
186
- logger.info("=======");
187
- sortedResults.forEach(({ title, result }) => {
188
- logger.info(`${title} => ${result}`);
189
- });
190
- if (githubSummary) {
191
- await (0, github_summary_utils_1.writeGitHubSummary)({ testRunUrl: testRunUrl, results });
192
- }
193
- return {
194
- testRun: {
195
- url: testRunUrl,
196
- id: testRun.id,
197
- project: testRun.project,
198
- status: overallStatus,
199
- progress: {
200
- flakedTestCases: sortedResults.filter(({ result }) => result === "flake").length,
201
- passedTestCases: sortedResults.filter(({ result }) => result === "pass")
202
- .length,
203
- failedTestCases: sortedResults.filter(({ result }) => result === "fail")
204
- .length,
205
- runningTestCases: 0,
206
- },
207
- },
208
- testCaseResults: sortedResults,
209
- };
210
- };
211
- exports.runAllTests = runAllTests;
212
- const getTestCasesWithBaseTestRunId = async ({ logger, client, baseCommitSha, baseTestRunId, testCases, }) => {
213
- const defaultBaseTestRunId = baseCommitSha != null
214
- ? await (0, test_run_api_1.getLatestTestRunId)({
215
- client,
216
- commitSha: baseCommitSha,
217
- })
218
- : null;
219
- const testsToRun = testCases.map((test) => {
220
- // We use the baseTestRunId specified in the test case if it exists, otherwise we use
221
- // use the baseTestRunId specified from the CLI args if it exists, otherwise we use the
222
- // baseTestRunId for the base commit if it exists, otherwise we use null (don't compare screenshots).
223
- const fallbackTestRunId = baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : defaultBaseTestRunId;
224
- if (test.baseTestRunId != null || fallbackTestRunId == null) {
225
- return test;
226
- }
227
- return { ...test, baseTestRunId: fallbackTestRunId };
228
- });
229
- if (baseCommitSha != null) {
230
- testsToRun
231
- .filter((test) => test.baseTestRunId == null)
232
- .forEach((test) => {
233
- logger.warn(`Skipping comparisons for test "${test.title}" since no result to compare against stored for base commit ${baseCommitSha}`);
234
- });
235
- }
236
- return testsToRun;
237
- };
@@ -1,6 +0,0 @@
1
- export interface TestRunProgress {
2
- failedTestCases: number;
3
- flakedTestCases: number;
4
- passedTestCases: number;
5
- runningTestCases: number;
6
- }
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
@@ -1,7 +0,0 @@
1
- import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
2
- import { DetailedTestCaseResult } from "../config/config.types";
3
- export type ScreenshotDiffResultWithBaseReplayId = ScreenshotDiffResult & {
4
- baseReplayId: string;
5
- };
6
- export declare const flattenScreenshotDiffResults: (testCaseResult: DetailedTestCaseResult) => ScreenshotDiffResultWithBaseReplayId[];
7
- export declare const groupScreenshotDiffResults: (results: ScreenshotDiffResultWithBaseReplayId[]) => Record<string, ScreenshotDiffResult[]>;
@@ -1,20 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.groupScreenshotDiffResults = exports.flattenScreenshotDiffResults = void 0;
4
- const flattenScreenshotDiffResults = (testCaseResult) => {
5
- return Object.entries(testCaseResult.screenshotDiffResultsByBaseReplayId).flatMap(([baseReplayId, diffs]) => {
6
- return diffs.map((diff) => ({ ...diff, baseReplayId }));
7
- });
8
- };
9
- exports.flattenScreenshotDiffResults = flattenScreenshotDiffResults;
10
- const groupScreenshotDiffResults = (results) => {
11
- const groupedResults = {};
12
- results.forEach(({ baseReplayId, ...result }) => {
13
- var _a;
14
- const resultsForBaseReplayId = (_a = groupedResults[baseReplayId]) !== null && _a !== void 0 ? _a : [];
15
- resultsForBaseReplayId.push(result);
16
- groupedResults[baseReplayId] = resultsForBaseReplayId;
17
- });
18
- return groupedResults;
19
- };
20
- exports.groupScreenshotDiffResults = groupScreenshotDiffResults;
@@ -1 +0,0 @@
1
- export {};
@@ -1,86 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- const common_1 = require("@alwaysmeticulous/common");
30
- const Sentry = __importStar(require("@sentry/node"));
31
- const loglevel_1 = __importDefault(require("loglevel"));
32
- const luxon_1 = require("luxon");
33
- const logger_utils_1 = require("../utils/logger.utils");
34
- const sentry_utils_1 = require("../utils/sentry.utils");
35
- const parallel_replay_handler_1 = require("./parallel-replay.handler");
36
- const INIT_TIMEOUT = luxon_1.Duration.fromObject({ second: 1 });
37
- const waitForInitMessage = () => {
38
- return Promise.race([
39
- new Promise((resolve) => {
40
- const messageHandler = (message) => {
41
- if (message &&
42
- typeof message === "object" &&
43
- message["kind"] === "init") {
44
- const initMessage = message;
45
- resolve(initMessage);
46
- process.off("message", messageHandler);
47
- }
48
- };
49
- process.on("message", messageHandler);
50
- }),
51
- new Promise((_resolve, reject) => {
52
- setTimeout(() => {
53
- reject(new Error("Timed out waiting for init message"));
54
- }, INIT_TIMEOUT.toMillis());
55
- }),
56
- ]);
57
- };
58
- const main = async () => {
59
- (0, logger_utils_1.initLogger)();
60
- await (0, sentry_utils_1.initSentry)();
61
- const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
62
- if (!process.send) {
63
- console.error("Error: not started as a child process");
64
- process.exit(1);
65
- }
66
- const initMessage = await waitForInitMessage();
67
- const { logLevel, dataDir, replayOptions } = initMessage.data;
68
- logger.setLevel(logLevel);
69
- (0, common_1.setMeticulousLocalDataDir)(dataDir);
70
- const result = await (0, parallel_replay_handler_1.handleReplay)(replayOptions);
71
- const resultMessage = {
72
- kind: "result",
73
- data: {
74
- result,
75
- },
76
- };
77
- process.send(resultMessage);
78
- process.disconnect();
79
- await Sentry.flush(sentry_utils_1.SENTRY_FLUSH_TIMEOUT.toMillis());
80
- };
81
- main().catch(async (error) => {
82
- console.error(error);
83
- Sentry.captureException(error);
84
- await Sentry.flush(sentry_utils_1.SENTRY_FLUSH_TIMEOUT.toMillis());
85
- process.exit(1);
86
- });
@@ -1 +0,0 @@
1
- export declare const getApiToken: (apiToken: string | null | undefined) => string | null;
@@ -1,31 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.getApiToken = void 0;
7
- const fs_1 = require("fs");
8
- const os_1 = require("os");
9
- const path_1 = require("path");
10
- const common_1 = require("@alwaysmeticulous/common");
11
- const loglevel_1 = __importDefault(require("loglevel"));
12
- const PERSONAL_CONFIG_FILE_PATH = ".meticulous/config.json";
13
- const getApiToken = (apiToken) => {
14
- const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
15
- if (apiToken) {
16
- return apiToken;
17
- }
18
- if (process.env["METICULOUS_API_TOKEN"]) {
19
- return process.env["METICULOUS_API_TOKEN"];
20
- }
21
- const personalConfigFileAbsolutePath = (0, path_1.join)((0, os_1.homedir)(), PERSONAL_CONFIG_FILE_PATH);
22
- if ((0, fs_1.existsSync)(personalConfigFileAbsolutePath)) {
23
- const config = JSON.parse((0, fs_1.readFileSync)(personalConfigFileAbsolutePath).toString("utf-8"));
24
- if (config.apiToken) {
25
- logger.info(`Using apiToken from ${personalConfigFileAbsolutePath}`);
26
- return config.apiToken;
27
- }
28
- }
29
- return null;
30
- };
31
- exports.getApiToken = getApiToken;
@@ -1 +0,0 @@
1
- export declare const getCommitSha: (commitSha: string | null | undefined) => Promise<string>;