@brillout/test-e2e 0.4.11 → 0.4.13

package/dist/Logs.d.ts

@@ -2,9 +2,9 @@ export { expectError };
 export declare const Logs: {
     add: typeof add;
     flushLogs: typeof flushLogs;
-    logErrors: typeof logErrors;
+    logErrorsAndWarnings: typeof logErrorsAndWarnings;
     clearLogs: typeof clearLogs;
-    hasErrorLogs: typeof hasErrorLogs;
+    hasFailLogs: typeof hasFailLogs;
     logEagerly: boolean;
 };
 declare type LogSource = 'stdout' | 'stderr' | 'Browser Error' | 'Browser Warning' | 'Browser Log' | 'Playwright' | 'run()' | 'run() failure' | 'test()' | 'Connection Error';
@@ -14,9 +14,9 @@ declare type LogEntry = {
     logTimestamp: string;
     isNotFailure: boolean;
 };
-declare function hasErrorLogs(failOnWarning: boolean): boolean;
+declare function hasFailLogs(failOnWarning: boolean): boolean;
 declare function clearLogs(): void;
-declare function logErrors(failOnWarning: boolean): void;
+declare function logErrorsAndWarnings(): void;
 declare function flushLogs(): void;
 declare function add({ logSource, logText }: {
     logSource: LogSource;

package/dist/Logs.js

@@ -7,9 +7,9 @@ exports.Logs = exports.expectError = void 0;
 exports.Logs = {
     add,
     flushLogs,
-    logErrors,
+    logErrorsAndWarnings,
     clearLogs,
-    hasErrorLogs,
+    hasFailLogs,
     logEagerly: false,
 };
 const utils_1 = require("./utils");
@@ -18,11 +18,11 @@ const getCurrentTest_1 = require("./getCurrentTest");
 const getConfig_1 = require("./getConfig");
 const logSection_1 = require("./logSection");
 let logEntries = [];
-function hasErrorLogs(failOnWarning) {
-    const errorLogs = getErrorLogs(failOnWarning);
-    return errorLogs.length > 0;
+function hasFailLogs(failOnWarning) {
+    const failLogs = getErrorLogs(failOnWarning);
+    return failLogs.length > 0;
 }
-function getErrorLogs(failOnWarning) {
+function getErrorLogs(returnWarnings) {
     const errorLogs = logEntries.filter(({ logSource, isNotFailure }) => {
         if (isNotFailure) {
             return;
@@ -30,7 +30,7 @@ function getErrorLogs(failOnWarning) {
         if (logSource === 'run() failure') {
             return true;
         }
-        if (failOnWarning && (logSource === 'Browser Warning' || logSource === 'stderr')) {
+        if (returnWarnings && (logSource === 'Browser Warning' || logSource === 'stderr')) {
             return true;
         }
         if (logSource === 'Browser Error') {
@@ -43,12 +43,12 @@ function getErrorLogs(failOnWarning) {
 function clearLogs() {
     logEntries = [];
 }
-function logErrors(failOnWarning) {
-    if (!exports.Logs.hasErrorLogs(failOnWarning))
+function logErrorsAndWarnings() {
+    const errorAndWarningLogs = getErrorLogs(true);
+    if (errorAndWarningLogs.length === 0)
         return;
-    (0, logSection_1.logSection)(`ERROR${!failOnWarning ? '' : '/WARNING'} LOGS`);
-    const errorLogs = getErrorLogs(failOnWarning);
-    errorLogs.forEach((logEntry) => printLog(logEntry));
+    (0, logSection_1.logSection)('ERROR & WARNING LOGS');
+    errorAndWarningLogs.forEach((logEntry) => printLog(logEntry));
 }
 function flushLogs() {
     (0, logSection_1.logSection)('ALL LOGS');

package/dist/cli.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const source_map_support_1 = __importDefault(require("source-map-support"));
 const utils_1 = require("./utils");
-const runTests_1 = require("./runTests");
+const runAll_1 = require("./runAll");
 const buildTs_1 = require("./buildTs");
 const Logs_1 = require("./Logs");
 initSourceMap();
@@ -14,7 +14,7 @@ initUserExitHandler();
 cli();
 function cli() {
     const filter = (0, utils_1.findFilesParseCliArgs)();
-    (0, runTests_1.runTests)(filter);
+    (0, runAll_1.runAll)(filter);
 }
 function initPromiseRejectionHandler() {
     process.on('unhandledRejection', function (err) {

package/dist/getCurrentTest.d.ts

@@ -19,6 +19,7 @@ declare type TestInfo = {
         inspect: boolean;
         doNotFailOnWarning: boolean;
         serverUrl: string;
+        isFlaky: boolean;
     };
     hasStartedRunning?: boolean;
     skipped?: string;

package/dist/logTestStatus.d.ts

@@ -0,0 +1,8 @@
+export { logPass };
+export { logWarn };
+export { logFail };
+export { hasFail };
+declare function logPass(): void;
+declare function logWarn(warning: string): void;
+declare function logFail(reason: string, isFinalAttempt: boolean): void;
+declare function hasFail(): boolean;

package/dist/logTestStatus.js

@@ -0,0 +1,56 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasFail = exports.logFail = exports.logWarn = exports.logPass = void 0;
+const getCurrentTest_1 = require("./getCurrentTest");
+const utils_1 = require("./utils");
+const picocolors_1 = __importDefault(require("picocolors"));
+function logPass() {
+    logStatus(false);
+}
+exports.logPass = logPass;
+function logWarn(warning) {
+    logStatus(false, warning);
+}
+exports.logWarn = logWarn;
+function logFail(reason, isFinalAttempt) {
+    if (isFinalAttempt) {
+        logStatus(true);
+    }
+    else {
+        logStatus(false, "Failed but marked as `isFlaky`; it'll be re-tried at the end.");
+    }
+    const { FAIL } = getStatusTags();
+    const color = (s) => picocolors_1.default.red(picocolors_1.default.bold(s));
+    const msg = `Test ${isFinalAttempt ? FAIL : 'failed'} because ${reason}, see below.`;
+    console.log(color(msg));
+}
+exports.logFail = logFail;
+function logStatus(isFail, warning) {
+    const { testFile } = (0, getCurrentTest_1.getCurrentTest)();
+    const { PASS, FAIL, WARN } = getStatusTags();
+    if (warning) {
+        (0, utils_1.assert)(!isFail);
+        console.log(`${WARN} ${testFile} (${warning})`);
+    }
+    else if (isFail) {
+        hasFailLog = true;
+        console.log(`${FAIL} ${testFile}`);
+    }
+    else {
+        console.log(`${PASS} ${testFile}`);
+    }
+}
+function getStatusTags() {
+    const PASS = picocolors_1.default.bold(picocolors_1.default.white(picocolors_1.default.bgGreen(' PASS ')));
+    const FAIL = picocolors_1.default.bold(picocolors_1.default.white(picocolors_1.default.bgRed(' FAIL ')));
+    const WARN = picocolors_1.default.bold(picocolors_1.default.white(picocolors_1.default.bgYellow(' WARN ')));
+    return { PASS, FAIL, WARN };
+}
+let hasFailLog = false;
+function hasFail() {
+    return hasFailLog;
+}
+exports.hasFail = hasFail;

package/dist/runAll.d.ts

@@ -0,0 +1,3 @@
+export { runAll };
+import { type FindFilter } from './utils';
+declare function runAll(filter: null | FindFilter): Promise<void>;

package/dist/{runTests.js → runAll.js}

@@ -1,14 +1,10 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.logFailure = exports.runTests = void 0;
+exports.runAll = void 0;
 const getCurrentTest_1 = require("./getCurrentTest");
 const Logs_1 = require("./Logs");
 const utils_1 = require("./utils");
 const utils_2 = require("./utils");
-const picocolors_1 = __importDefault(require("picocolors"));
 const github_action_1 = require("./github-action");
 const getCurrentTest_2 = require("./getCurrentTest");
 const getBrowser_1 = require("./getBrowser");
@@ -16,54 +12,68 @@ const buildTs_1 = require("./buildTs");
 const findTestFiles_1 = require("./findTestFiles");
 const getConfig_1 = require("./getConfig");
 const logError_1 = require("./logError");
-let hasFailure = false;
-async function runTests(filter) {
+const logTestStatus_1 = require("./logTestStatus");
+async function runAll(filter) {
     await (0, getConfig_1.loadConfig)();
     const testFiles = await (0, findTestFiles_1.findTestFiles)(filter);
     const browser = await (0, getBrowser_1.getBrowser)();
-    let hasFailedTest = false;
+    const failedFirstAttempt = [];
     for (const testFile of testFiles) {
-        (0, utils_1.assert)(testFile.endsWith('.ts'));
-        const testFileJs = testFile.replace('.ts', '.mjs');
-        const clean = await (0, buildTs_1.buildTs)(testFile, testFileJs);
-        (0, getCurrentTest_2.setCurrentTest)(testFile);
-        try {
-            await import((0, utils_2.fsWindowsBugWorkaround)(testFileJs));
-        }
-        finally {
-            clean();
+        const success = await buildAndTest(testFile, browser, false);
+        if (!success) {
+            failedFirstAttempt.push(testFile);
         }
-        const success = await runTestFile(browser);
+    }
+    const failedSecondAttempt = [];
+    for (const testFile of failedFirstAttempt) {
+        const success = await buildAndTest(testFile, browser, true);
         if (!success) {
-            hasFailedTest = true;
+            failedSecondAttempt.push(testFile);
         }
-        (0, getCurrentTest_2.setCurrentTest)(null);
-        (0, utils_1.assert)(testFileJs.endsWith('.mjs'));
     }
     await browser.close();
-    if (hasFailedTest || hasFailure) {
-        // hasFailedTest and hasFailure are redundant
-        //  - When assert.ts calls logFailure() this code block isn't run
-        (0, utils_1.assert)(hasFailedTest && hasFailure);
-        throw new Error('A test failed. See messages above.');
+    const hasFailLog = (0, logTestStatus_1.hasFail)();
+    const hasFailedTestFile = failedSecondAttempt.length > 0;
+    if (hasFailedTestFile || hasFailLog) {
+        // hasFailedTestFile and hasFailLog are redundant
+        //  - When assert.ts calls logFail() this code block isn't run
+        (0, utils_1.assert)(hasFailedTestFile && hasFailLog);
+        throw new Error('Following tests failed, see logs above for more information.');
+    }
+}
+exports.runAll = runAll;
+async function buildAndTest(testFile, browser, isSecondAttempt) {
+    (0, utils_1.assert)(testFile.endsWith('.ts'));
+    const testFileJs = testFile.replace('.ts', '.mjs');
+    const cleanBuild = await (0, buildTs_1.buildTs)(testFile, testFileJs);
+    (0, getCurrentTest_2.setCurrentTest)(testFile);
+    try {
+        await import((0, utils_2.fsWindowsBugWorkaround)(testFileJs) + `?cacheBuster=${Date.now()}`);
+    }
+    finally {
+        cleanBuild();
     }
+    const { success, clean } = await runTests(browser, isSecondAttempt);
+    await clean();
+    (0, getCurrentTest_2.setCurrentTest)(null);
+    (0, utils_1.assert)(testFileJs.endsWith('.mjs'));
+    return success;
 }
-exports.runTests = runTests;
-async function runTestFile(browser) {
+async function runTests(browser, isSecondAttempt) {
     const testInfo = (0, getCurrentTest_1.getCurrentTest)();
     if (utils_1.isTTY) {
         console.log();
         console.log(testInfo.testFile);
     }
-    const page = await browser.newPage();
-    testInfo.page = page;
     // Set when user calls `skip()`
     if (testInfo.skipped) {
-        logTestStatus(false);
+        (0, logTestStatus_1.logWarn)(testInfo.skipped);
         (0, utils_1.assertUsage)(!testInfo.runInfo, 'You cannot call `run()` after calling `skip()`');
         (0, utils_1.assertUsage)(testInfo.tests === undefined, 'You cannot call `test()` after calling `skip()`');
-        return true;
+        return { success: true, clean: async () => { } };
     }
+    const page = await browser.newPage();
+    testInfo.page = page;
     // Set when user calls `run()`
     (0, utils_1.assert)(testInfo.runInfo);
     (0, utils_1.assert)(testInfo.startServer);
@@ -71,24 +81,23 @@ async function runTestFile(browser) {
     (0, utils_1.assert)(testInfo.afterEach);
     // Set when user calls `test()`
     (0, utils_1.assert)(testInfo.tests);
+    const isFinalAttempt = isSecondAttempt || !testInfo.runInfo.isFlaky;
     const clean = async () => {
         await testInfo.terminateServer?.();
         await page.close();
     };
-    const abort = async () => {
-        await clean();
+    const failure = () => {
         (0, github_action_1.abortIfGitHubAction)();
+        return { success: false, clean };
     };
-    // TODO: resolve a success flag instead rejecting
     try {
         await testInfo.startServer();
     }
     catch (err) {
-        logFailure('an error occurred while starting the server');
+        (0, logTestStatus_1.logFail)('an error occurred while starting the server', isFinalAttempt);
         (0, logError_1.logError)(err);
         Logs_1.Logs.flushLogs();
-        await abort();
-        return false;
+        return failure();
     }
     for (const { testDesc, testFn } of testInfo.tests) {
         Logs_1.Logs.add({
@@ -107,44 +116,41 @@ async function runTestFile(browser) {
         testInfo.afterEach(!!err);
         {
             const failOnWarning = !testInfo.runInfo.doNotFailOnWarning;
-            const hasErrorLog = Logs_1.Logs.hasErrorLogs(failOnWarning);
+            const hasErrorLog = Logs_1.Logs.hasFailLogs(failOnWarning);
             const isFailure = err || hasErrorLog;
             if (isFailure) {
                 if (err) {
-                    logFailure(`the test "${testDesc}" threw an error`);
+                    (0, logTestStatus_1.logFail)(`the test "${testDesc}" threw an error`, isFinalAttempt);
                     (0, logError_1.logError)(err);
                 }
                 else if (hasErrorLog) {
-                    logFailure(`${getErrorType(failOnWarning)} occurred while running the test "${testDesc}"`);
+                    (0, logTestStatus_1.logFail)(`${getErrorType(failOnWarning)} occurred while running the test "${testDesc}"`, isFinalAttempt);
                 }
                 else {
                     (0, utils_1.assert)(false);
                 }
-                Logs_1.Logs.logErrors(failOnWarning);
+                Logs_1.Logs.logErrorsAndWarnings();
                 Logs_1.Logs.flushLogs();
-                await abort();
-                return false;
+                return failure();
             }
         }
         Logs_1.Logs.clearLogs();
     }
-    await clean();
     // Check whether stderr emitted during testInfo.terminateServer()
     {
         const failOnWarning = true;
-        if (Logs_1.Logs.hasErrorLogs(failOnWarning) &&
+        if (Logs_1.Logs.hasFailLogs(failOnWarning) &&
             // See comments about taskkill in src/setup.ts
             !(0, utils_1.isWindows)()) {
-            logFailure(`${getErrorType(failOnWarning)} occurred during server termination`);
-            Logs_1.Logs.logErrors(failOnWarning);
+            (0, logTestStatus_1.logFail)(`${getErrorType(failOnWarning)} occurred during server termination`, isFinalAttempt);
+            Logs_1.Logs.logErrorsAndWarnings();
             Logs_1.Logs.flushLogs();
-            await abort();
-            return false;
+            return failure();
         }
     }
     Logs_1.Logs.clearLogs();
-    logTestStatus(true);
-    return true;
+    (0, logTestStatus_1.logPass)();
+    return { success: true, clean };
 }
 function getErrorType(failOnWarning) {
     return !failOnWarning ? 'error(s)' : 'error(s)/warning(s)';
@@ -174,33 +180,3 @@ function runTest(testFn, testFunctionTimeout) {
     })();
     return promise;
 }
-function logTestStatus(success) {
-    const testInfo = (0, getCurrentTest_1.getCurrentTest)();
-    const { PASS, FAIL, WARN } = getStatusTags();
-    if (success) {
-        (0, utils_1.assert)(!testInfo.skipped);
-        console.log(`${PASS} ${testInfo.testFile}`);
-        return;
-    }
-    if (testInfo.skipped) {
-        console.log(`${WARN} ${testInfo.testFile} (${testInfo.skipped})`);
-    }
-    else {
-        console.log(`${FAIL} ${testInfo.testFile}`);
-    }
-}
-function logFailure(reason) {
-    hasFailure = true;
-    logTestStatus(false);
-    const { FAIL } = getStatusTags();
-    const color = (s) => picocolors_1.default.red(picocolors_1.default.bold(s));
-    const msg = `Test ${FAIL} because ${reason}, see below.`;
-    console.log(color(msg));
-}
-exports.logFailure = logFailure;
-function getStatusTags() {
-    const PASS = picocolors_1.default.bold(picocolors_1.default.bgGreen(picocolors_1.default.white(' PASS ')));
-    const FAIL = picocolors_1.default.bold(picocolors_1.default.bgRed(picocolors_1.default.white(' FAIL ')));
-    const WARN = picocolors_1.default.bold(picocolors_1.default.bgYellow(picocolors_1.default.white(' WARN ')));
-    return { PASS, FAIL, WARN };
-}

package/dist/setup.d.ts

@@ -15,7 +15,7 @@ export { isMac };
 export { sleep };
 export { getServerUrl };
 declare function skip(reason: string): void;
-declare function run(cmd: string, { additionalTimeout, serverIsReadyMessage, serverIsReadyDelay, inspect, cwd, doNotFailOnWarning, serverUrl, }?: {
+declare function run(cmd: string, { additionalTimeout, serverIsReadyMessage, serverIsReadyDelay, inspect, cwd, doNotFailOnWarning, serverUrl, isFlaky, }?: {
     additionalTimeout?: number;
     serverIsReadyMessage?: string | ((log: string) => boolean);
     serverIsReadyDelay?: number;
@@ -23,6 +23,7 @@ declare function run(cmd: string, { additionalTimeout, serverIsReadyMessage, ser
     cwd?: string;
     doNotFailOnWarning?: boolean;
     serverUrl?: string;
+    isFlaky?: boolean;
 }): void;
 declare function autoRetry(test: () => void | Promise<void>, { timeout }?: {
     timeout?: number;

package/dist/setup.js

@@ -36,7 +36,7 @@ function skip(reason) {
 exports.skip = skip;
 function run(cmd, { 
 //baseUrl = '',
-additionalTimeout = 0, serverIsReadyMessage, serverIsReadyDelay = 1000, inspect = utils_1.cliConfig.inspect, cwd, doNotFailOnWarning = false, serverUrl = serverUrlDefault, } = {}) {
+additionalTimeout = 0, serverIsReadyMessage, serverIsReadyDelay = 1000, inspect = utils_1.cliConfig.inspect, cwd, doNotFailOnWarning = false, serverUrl = serverUrlDefault, isFlaky = false, } = {}) {
     additionalTimeout += serverIsReadyDelay;
     const testInfo = (0, getCurrentTest_1.getCurrentTest)();
     testInfo.runInfo = {
@@ -49,6 +49,7 @@ additionalTimeout = 0, serverIsReadyMessage, serverIsReadyDelay = 1000, inspect
         inspect,
         doNotFailOnWarning,
         serverUrl,
+        isFlaky,
     };
     if (inspect) {
         Logs_1.Logs.logEagerly = true;

package/dist/utils/assert.js

@@ -1,19 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.projectErrorPrefix = exports.getProjectError = exports.assertInfo = exports.assertWarning = exports.assertUsage = exports.assert = void 0;
-const runTests_1 = require("../runTests");
 const projectInfo_1 = require("./projectInfo");
 const Logs_1 = require("../Logs");
 const getCurrentTest_1 = require("../getCurrentTest");
 const logSection_1 = require("../logSection");
 const logError_1 = require("../logError");
+const logTestStatus_1 = require("../logTestStatus");
 const errorPrefix = `[${projectInfo_1.projectInfo.npmPackageName}]`;
 exports.projectErrorPrefix = errorPrefix;
 const internalErrorPrefix = `${errorPrefix}[Bug]`;
 const usageErrorPrefix = `${errorPrefix}[Wrong Usage]`;
 const warningPrefix = `${errorPrefix}[Warning]`;
 const infoPrefix = `${errorPrefix}[Info]`;
-const numberOfStackTraceLinesToRemove = 2;
 function assert(condition, debugInfo) {
     if (condition) {
         return;
@@ -32,10 +31,10 @@ function assert(condition, debugInfo) {
         `Do not hesitate to reach out as it makes ${projectInfo_1.projectInfo.projectName} more robust.`,
         debugStr,
     ].join(' '));
-    (0, runTests_1.logFailure)('a bug occurred in @brillout/test-e2e');
+    (0, logTestStatus_1.logFail)('a bug occurred in @brillout/test-e2e', true);
     (0, logError_1.logError)(internalError, 'BUG');
     logDebugInfo();
-    Logs_1.Logs.logErrors(true);
+    Logs_1.Logs.logErrorsAndWarnings();
     Logs_1.Logs.flushLogs();
     throw new Error('Bug. See messages above.');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brillout/test-e2e",
-  "version": "0.4.11",
+  "version": "0.4.13",
   "bin": "./bin.js",
   "main": "./dist/index.js",
   "scripts": {

package/dist/runTests.d.ts

@@ -1,5 +0,0 @@
-export { runTests };
-export { logFailure };
-import { type FindFilter } from './utils';
-declare function runTests(filter: null | FindFilter): Promise<void>;
-declare function logFailure(reason: string): void;