@alwaysmeticulous/cli 2.21.1 → 2.21.3-alpha.1

package/dist/commands/screenshot-diff/screenshot-diff.command.js

@@ -51,8 +51,8 @@ const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreens
         const headScreenshotFile = (0, path_1.join)(headScreenshotsDir, screenshotFileName);
         const baseScreenshot = await (0, io_utils_1.readPng)(baseScreenshotFile);
         const headScreenshot = await (0, io_utils_1.readPng)(headScreenshotFile);
-        if (baseScreenshot.width !== baseScreenshot.width ||
-            headScreenshot.height !== headScreenshot.height) {
+        if (baseScreenshot.width !== headScreenshot.width ||
+            baseScreenshot.height !== headScreenshot.height) {
             return [
                 {
                     identifier,

package/dist/parallel-tests/__tests__/merge-test-results.spec.js

@@ -1,191 +1,102 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const merge_test_results_1 = require("../merge-test-results");
+const mock_test_results_1 = require("./mock-test-results");
 describe("mergeResults", () => {
     it("keeps the result as a failure when all retried screenshots are 'identical'", () => {
-        const currentResult = testResult("fail", [diff(0), noDiff(1)]);
-        const comparisonToHeadReplay = testResult("pass", [noDiff(0), noDiff(1)]);
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.noDiff)(1)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("fail", [diff(0), noDiff(1)]));
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.noDiff)(1)]));
     });
     it("ignores diffs to screenshots which originally passed", () => {
-        const currentResult = testResult("pass", [noDiff(0), noDiff(1)]);
-        const comparisonToHeadReplay = testResult("fail", [noDiff(0), diff(1)]);
+        const currentResult = (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("pass", [noDiff(0), noDiff(1)]));
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)]));
     });
     it("marks screenshots as flakes if the screenshot comparison originally failed, but the second retry gives a different screenshot again", () => {
-        const currentResult = testResult("fail", [noDiff(0), diff(1), diff(2)]);
-        const comparisonToHeadReplay = testResult("fail", [
-            noDiff(0),
-            diff(1),
-            noDiff(2),
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1), (0, mock_test_results_1.diff)(2)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.noDiff)(0),
+            (0, mock_test_results_1.diff)(1),
+            (0, mock_test_results_1.noDiff)(2),
         ]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("fail", [noDiff(0), flake(1, diff(), [diff()]), diff(2)]));
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)()]), (0, mock_test_results_1.diff)(2)]));
     });
     it("marks overall test as a flake if there are only flakey screenshots and no failed screenshots", () => {
-        const currentResult = testResult("fail", [noDiff(0), diff(1), diff(2)]);
-        const comparisonToHeadReplay = testResult("fail", [
-            noDiff(0),
-            diff(1),
-            diff(2),
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1), (0, mock_test_results_1.diff)(2)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.noDiff)(0),
+            (0, mock_test_results_1.diff)(1),
+            (0, mock_test_results_1.diff)(2),
         ]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("flake", [
-            noDiff(0),
-            flake(1, diff(), [diff()]),
-            flake(2, diff(), [diff()]),
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("flake", [
+            (0, mock_test_results_1.noDiff)(0),
+            (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)()]),
+            (0, mock_test_results_1.flake)(2, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)()]),
         ]));
     });
     it("adds to diffsToHeadScreenshotOnRetries for existing flakes", () => {
-        const currentResult = testResult("fail", [
-            diff(0),
-            flake(1, diff(), [missingHead()]),
-            flake(2, differentSize(), [diff()]),
-            flake(3, missingBase(), [diff(), diff()]),
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.diff)(0),
+            (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)()]),
+            (0, mock_test_results_1.flake)(2, (0, mock_test_results_1.differentSize)(), [(0, mock_test_results_1.diff)()]),
+            (0, mock_test_results_1.flake)(3, (0, mock_test_results_1.missingBase)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()]),
         ]);
-        const comparisonToHeadReplay = testResult("fail", [
-            noDiff(0),
-            diff(1),
-            differentSize(2),
-            missingHead(3),
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.noDiff)(0),
+            (0, mock_test_results_1.diff)(1),
+            (0, mock_test_results_1.differentSize)(2),
+            (0, mock_test_results_1.missingHead)(3),
         ]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("fail", [
-            diff(0),
-            flake(1, diff(), [missingHead(), diff()]),
-            flake(2, differentSize(), [diff(), differentSize()]),
-            flake(3, missingBase(), [diff(), diff(), missingHead()]),
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.diff)(0),
+            (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)(), (0, mock_test_results_1.diff)()]),
+            (0, mock_test_results_1.flake)(2, (0, mock_test_results_1.differentSize)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.differentSize)()]),
+            (0, mock_test_results_1.flake)(3, (0, mock_test_results_1.missingBase)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.missingHead)()]),
         ]));
     });
     it("keeps a missing-head as is, if there is no corresponding retry screenshot", () => {
-        const currentResult = testResult("fail", [missingHead(0)]);
-        const comparisonToHeadReplay = testResult("pass", []);
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("pass", []);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("fail", [missingHead(0)]));
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0)]));
     });
     it("adds to diffsToHeadScreenshotOnRetries for a flakey missing-head, if there is no corresponding retry screenshot", () => {
-        const currentResult = testResult("fail", [
-            flake(0, missingHead(), [missingBase()]),
-            diff(1),
+        const currentResult = (0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.flake)(0, (0, mock_test_results_1.missingHead)(), [(0, mock_test_results_1.missingBase)()]),
+            (0, mock_test_results_1.diff)(1),
         ]);
-        const comparisonToHeadReplay = testResult("pass", [noDiff(1)]);
+        const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(1)]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual(testResult("fail", [
-            flake(0, missingHead(), [missingBase(), missingBaseAndHead()]),
-            diff(1),
+        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [
+            (0, mock_test_results_1.flake)(0, (0, mock_test_results_1.missingHead)(), [(0, mock_test_results_1.missingBase)(), (0, mock_test_results_1.missingBaseAndHead)()]),
+            (0, mock_test_results_1.diff)(1),
         ]));
     });
 });
-const id = (eventNumber = 0) => ({
-    type: "after-event",
-    eventNumber,
-});
-const testResult = (result, screenshotDiffResults) => {
-    return {
-        sessionId: "mock-session-id",
-        headReplayId: "mock-head-replay-id",
-        result,
-        screenshotDiffResults,
-    };
-};
-const diff = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "diff",
-        headScreenshotFile: "mock-head-file",
-        baseScreenshotFile: "mock-base-file",
-        width: 1920,
-        height: 1080,
-        mismatchFraction: 0.01,
-        mismatchPixels: 1000,
-    };
-};
-const noDiff = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "no-diff",
-        headScreenshotFile: "mock-head-file",
-        baseScreenshotFile: "mock-base-file",
-        width: 1920,
-        height: 1080,
-        mismatchFraction: 0.01,
-        mismatchPixels: 1000,
-    };
-};
-const flake = (eventNumber, diffToBaseScreenshot, diffsToHeadScreenshotOnRetries) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "flake",
-        diffToBaseScreenshot: asSingleTryDiff(diffToBaseScreenshot),
-        diffsToHeadScreenshotOnRetries: diffsToHeadScreenshotOnRetries.map(asRetryDiff),
-    };
-};
-const missingBase = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "missing-base",
-        headScreenshotFile: "mock-head-file",
-    };
-};
-const missingHead = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "missing-head",
-        baseScreenshotFile: "mock-base-file",
-    };
-};
-const missingBaseAndHead = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "missing-base-and-head",
-    };
-};
-const differentSize = (eventNumber) => {
-    return {
-        identifier: id(eventNumber),
-        outcome: "different-size",
-        baseScreenshotFile: "mock-base-file",
-        headScreenshotFile: "mock-head-file",
-        baseWidth: 1920,
-        baseHeight: 1080,
-        headWidth: 10,
-        headHeight: 5,
-    };
-};
-const asSingleTryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
-...rest }) => {
-    if (rest.outcome === "flake") {
-        throw new Error("Must not be a diff with a flake outcome");
-    }
-    return rest;
-};
-const asRetryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
-...rest }) => {
-    if (rest.outcome === "flake") {
-        throw new Error("Must not be a diff with a flake outcome");
-    }
-    return rest;
-};

package/dist/parallel-tests/__tests__/mock-test-results.d.ts

@@ -0,0 +1,17 @@
+import { ScreenshotDiffResult, ScreenshotIdentifier, TestCase } from "@alwaysmeticulous/api";
+import { DetailedTestCaseResult } from "../../config/config.types";
+export declare const id: (eventNumber?: number) => ScreenshotIdentifier;
+export declare const testResult: (result: "pass" | "fail" | "flake", screenshotDiffResults: ScreenshotDiffResult[], testCase?: TestCase) => DetailedTestCaseResult;
+export declare const diff: (eventNumber?: number) => ScreenshotDiffResult;
+export declare const noDiff: (eventNumber?: number) => ScreenshotDiffResult;
+export declare const flake: (eventNumber: number, diffToBaseScreenshot: ScreenshotDiffResult, diffsToHeadScreenshotOnRetries: Array<ScreenshotDiffResult | {
+    identifier: ScreenshotIdentifier;
+    outcome: "missing-base-and-head";
+}>) => ScreenshotDiffResult;
+export declare const missingBase: (eventNumber?: number) => ScreenshotDiffResult;
+export declare const missingHead: (eventNumber?: number) => ScreenshotDiffResult;
+export declare const missingBaseAndHead: (eventNumber?: number) => {
+    readonly identifier: ScreenshotIdentifier;
+    readonly outcome: "missing-base-and-head";
+};
+export declare const differentSize: (eventNumber?: number) => ScreenshotDiffResult;

package/dist/parallel-tests/__tests__/mock-test-results.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.differentSize = exports.missingBaseAndHead = exports.missingHead = exports.missingBase = exports.flake = exports.noDiff = exports.diff = exports.testResult = exports.id = void 0;
+const id = (eventNumber = 0) => ({
+    type: "after-event",
+    eventNumber,
+});
+exports.id = id;
+const testResult = (result, screenshotDiffResults, testCase) => {
+    var _a;
+    return {
+        ...testCase,
+        sessionId: (_a = testCase === null || testCase === void 0 ? void 0 : testCase.sessionId) !== null && _a !== void 0 ? _a : "mock-session-id",
+        headReplayId: "mock-head-replay-id",
+        result,
+        screenshotDiffResults,
+    };
+};
+exports.testResult = testResult;
+const diff = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "diff",
+        headScreenshotFile: "mock-head-file",
+        baseScreenshotFile: "mock-base-file",
+        width: 1920,
+        height: 1080,
+        mismatchFraction: 0.01,
+        mismatchPixels: 1000,
+    };
+};
+exports.diff = diff;
+const noDiff = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "no-diff",
+        headScreenshotFile: "mock-head-file",
+        baseScreenshotFile: "mock-base-file",
+        width: 1920,
+        height: 1080,
+        mismatchFraction: 0.01,
+        mismatchPixels: 1000,
+    };
+};
+exports.noDiff = noDiff;
+const flake = (eventNumber, diffToBaseScreenshot, diffsToHeadScreenshotOnRetries) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "flake",
+        diffToBaseScreenshot: asSingleTryDiff(diffToBaseScreenshot),
+        diffsToHeadScreenshotOnRetries: diffsToHeadScreenshotOnRetries.map(asRetryDiff),
+    };
+};
+exports.flake = flake;
+const missingBase = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "missing-base",
+        headScreenshotFile: "mock-head-file",
+    };
+};
+exports.missingBase = missingBase;
+const missingHead = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "missing-head",
+        baseScreenshotFile: "mock-base-file",
+    };
+};
+exports.missingHead = missingHead;
+const missingBaseAndHead = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "missing-base-and-head",
+    };
+};
+exports.missingBaseAndHead = missingBaseAndHead;
+const differentSize = (eventNumber) => {
+    return {
+        identifier: (0, exports.id)(eventNumber),
+        outcome: "different-size",
+        baseScreenshotFile: "mock-base-file",
+        headScreenshotFile: "mock-head-file",
+        baseWidth: 1920,
+        baseHeight: 1080,
+        headWidth: 10,
+        headHeight: 5,
+    };
+};
+exports.differentSize = differentSize;
+const asSingleTryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
+...rest }) => {
+    if (rest.outcome === "flake") {
+        throw new Error("Must not be a diff with a flake outcome");
+    }
+    return rest;
+};
+const asRetryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
+...rest }) => {
+    if (rest.outcome === "flake") {
+        throw new Error("Must not be a diff with a flake outcome");
+    }
+    return rest;
+};

package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.js

@@ -0,0 +1,175 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const common_1 = require("@alwaysmeticulous/common");
+const parallel_tests_handler_1 = require("../parallel-tests.handler");
+const mock_test_results_1 = require("./mock-test-results");
+describe("runAllTestsInParallel", () => {
+    it("maximises the number of tests run in parralel, up to the provided limit", async () => {
+        let progress = {
+            runningTestCases: 4,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 0,
+        };
+        let testsStarted = 0;
+        const testRunPromises = [(0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)()];
+        const overallPromise = (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 0,
+            executeTest: async (testCase) => {
+                var _a;
+                testsStarted++;
+                await testRunPromises[parseInt((_a = testCase.title) !== null && _a !== void 0 ? _a : "")].promise;
+                return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase);
+            },
+            onTestFinished: async (newProgress) => {
+                progress = newProgress;
+            },
+        });
+        await waitForPromisesToFlush();
+        expect(testsStarted).toEqual(2);
+        expect(progress).toEqual({
+            runningTestCases: 4,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 0,
+        });
+        testRunPromises[0].resolve();
+        await waitForPromisesToFlush();
+        expect(testsStarted).toEqual(3);
+        expect(progress).toEqual({
+            runningTestCases: 3,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 1,
+        });
+        testRunPromises[1].resolve();
+        testRunPromises[2].resolve();
+        await waitForPromisesToFlush();
+        expect(testsStarted).toEqual(4);
+        expect(progress).toEqual({
+            runningTestCases: 1,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 3,
+        });
+        await expectPromiseToNotHaveResolved(overallPromise);
+        testRunPromises[3].resolve();
+        await waitForPromisesToFlush();
+        expect(progress).toEqual({
+            runningTestCases: 0,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 4,
+        });
+        await overallPromise;
+    });
+    it("can handle a test failing to run", async () => {
+        let progress = {
+            runningTestCases: 4,
+            failedTestCases: 0,
+            flakedTestCases: 0,
+            passedTestCases: 0,
+        };
+        const testRunPromises = [(0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)(), (0, common_1.defer)()];
+        const overallPromise = (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 0,
+            executeTest: async (testCase) => {
+                var _a;
+                await testRunPromises[parseInt((_a = testCase.title) !== null && _a !== void 0 ? _a : "")].promise;
+                return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase);
+            },
+            onTestFinished: async (newProgress) => {
+                progress = newProgress;
+            },
+        });
+        testRunPromises[0].resolve();
+        testRunPromises[1].reject();
+        testRunPromises[2].resolve();
+        testRunPromises[3].resolve();
+        const result = await overallPromise;
+        expect(progress).toEqual({
+            runningTestCases: 0,
+            failedTestCases: 1,
+            flakedTestCases: 0,
+            passedTestCases: 3,
+        });
+        // No result given for test 1 (it can't, because it doesn't have a headReplayId)
+        expect(result).toEqual([
+            (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(0)),
+            (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(2)),
+            (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0)], testCase(3)),
+        ]);
+    });
+    it("does not retry failed tests if maxRetriesOnFailure is 0", async () => {
+        const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 0,
+            executeTest: async (testCase) => {
+                return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0)], testCase);
+            },
+        });
+        expect(results).toEqual([(0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0)], testCase(0))]);
+    });
+    it("retries failed tests until maxRetriesOnFailure reached or first flake seen for all screenshots if maxRetriesOnFailure > 0", async () => {
+        let retryNum = 0;
+        const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 10,
+            executeTest: async (testCase, isRetry) => {
+                if (!isRetry) {
+                    return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
+                }
+                if (retryNum === 3) {
+                    return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0), (0, mock_test_results_1.diff)(1)], testCase);
+                }
+                retryNum++;
+                return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)], testCase);
+            },
+        });
+        expect(retryNum).toEqual(3);
+        expect(results).toEqual([
+            (0, mock_test_results_1.testResult)("flake", [
+                (0, mock_test_results_1.flake)(0, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)()]),
+                (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()]),
+            ], testCase(0)),
+        ]);
+    });
+    it("marks test as a failure if hit max number of retries and no flakes detected", async () => {
+        const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 3,
+            executeTest: async (testCase, isRetry) => {
+                if (!isRetry) {
+                    return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
+                }
+                return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)], testCase);
+            },
+        });
+        expect(results).toEqual([
+            (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()])], testCase(0)),
+        ]);
+    });
+});
+const testCase = (num) => ({
+    sessionId: "mock-session-id",
+    title: `${num}`,
+    baseReplayId: "mock-base-replay-id",
+});
+const expectPromiseToNotHaveResolved = async (promise) => {
+    let resolved = false;
+    promise.then(() => {
+        resolved = true;
+    }, () => {
+        resolved = true;
+    });
+    await waitForPromisesToFlush();
+    expect(resolved).toEqual(false);
+};
+const waitForPromisesToFlush = () => new Promise((resolve) => setTimeout(resolve, 0));

package/dist/parallel-tests/execute-test-in-child-process.d.ts

@@ -0,0 +1,3 @@
+import { DetailedTestCaseResult } from "../config/config.types";
+import { InitMessage } from "./messages.types";
+export declare const executeTestInChildProcess: (initMessage: InitMessage) => Promise<DetailedTestCaseResult>;

package/dist/parallel-tests/execute-test-in-child-process.js

@@ -0,0 +1,43 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeTestInChildProcess = void 0;
+const child_process_1 = require("child_process");
+const path_1 = require("path");
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const executeTestInChildProcess = (initMessage) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const taskHandler = (0, path_1.join)(__dirname, "task.handler.js");
+    const deferredResult = (0, common_1.defer)();
+    const child = (0, child_process_1.fork)(taskHandler, [], { stdio: "inherit" });
+    const messageHandler = (message) => {
+        if (message &&
+            typeof message === "object" &&
+            message["kind"] === "result") {
+            const resultMessage = message;
+            deferredResult.resolve(resultMessage.data.result);
+            child.off("message", messageHandler);
+        }
+    };
+    child.on("error", (error) => {
+        if (deferredResult.getState() === "pending") {
+            deferredResult.reject(error);
+        }
+    });
+    child.on("exit", (code) => {
+        if (code) {
+            logger.debug(`child exited with code: ${code}`);
+        }
+        if (deferredResult.getState() === "pending") {
+            deferredResult.reject(new Error("No result"));
+        }
+    });
+    child.on("message", messageHandler);
+    // Send test case and arguments to child process
+    child.send(initMessage);
+    return deferredResult.promise;
+};
+exports.executeTestInChildProcess = executeTestInChildProcess;

package/dist/parallel-tests/parallel-tests.handler.d.ts

@@ -1,24 +1,13 @@
 import { TestCase } from "@alwaysmeticulous/api";
-import { ReplayEventsDependencies, ReplayExecutionOptions } from "@alwaysmeticulous/common";
-import { TestRun } from "../api/test-run.api";
-import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
-import { DetailedTestCaseResult, MeticulousCliConfig } from "../config/config.types";
+import { DetailedTestCaseResult } from "../config/config.types";
 import { TestRunProgress } from "./run-all-tests.types";
 export interface RunAllTestsInParallelOptions {
-    config: MeticulousCliConfig;
-    testRun: TestRun;
     testsToRun: TestCase[];
-    executionOptions: ReplayExecutionOptions;
-    screenshottingOptions: ScreenshotAssertionsEnabledOptions;
-    apiToken: string | null;
-    commitSha: string;
-    appUrl: string | null;
-    useAssetsSnapshottedInBaseSimulation: boolean;
     parallelTasks: number | null;
-    deflake: boolean;
-    replayEventsDependencies: ReplayEventsDependencies;
     maxRetriesOnFailure: number;
+    executeTest: (testCase: TestCase, isRetry: boolean) => Promise<DetailedTestCaseResult>;
     onTestFinished?: (progress: TestRunProgress, resultsSoFar: DetailedTestCaseResult[]) => Promise<void>;
+    onTestFailedToRun?: (progress: TestRunProgress) => Promise<void>;
 }
 /** Handler for running Meticulous tests in parallel using child processes */
 export declare const runAllTestsInParallel: (options: RunAllTestsInParallelOptions) => Promise<DetailedTestCaseResult[]>;