@testim/testim-cli 3.197.0 → 3.201.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testim/testim-cli",
-  "version": "3.197.0",
+  "version": "3.201.0",
   "description": "Command line interface for running Testing on your CI",
   "author": "Oren Rubin",
   "contributors": [{
@@ -36,7 +36,7 @@
     "typescript": "3.9.3"
   },
   "lazyDependencies": {
-    "ngrok": "3.2.3",
+    "ngrok": "3.4.0",
     "webpack": "4.43.0",
     "playwright": "0.12.1",
     "puppeteer": "2.1.1",
@@ -79,10 +79,10 @@
     "glob": "7.1.6",
     "istanbul-lib-report": "3.0.0",
     "istanbul-reports": "3.0.2",
-    "jimp": "0.2.28",
+    "jimp": "0.16.1",
     "jsdom": "16.7.0",
     "jsonwebtoken": "8.5.1",
-    "lodash": "4.17.20",
+    "lodash": "4.17.21",
     "memory-fs": "0.5.0",
     "memorystream": "0.3.1",
     "mkdirp": "1.0.4",
@@ -105,9 +105,9 @@
     "superagent-proxy": "3.0.0",
     "test-exclude": "6.0.0",
     "threads": "0.12.0",
-    "ua-parser-js": "0.7.22",
+    "ua-parser-js": "0.7.28",
     "validate-npm-package-name": "3.0.0",
-    "ws": "7.3.0",
+    "ws": "8.2.3",
     "xml2js": "0.4.23",
     "yaml": "1.10.0"
   },

package/player/stepActions/pixelValidationStepAction.js

@@ -7,9 +7,9 @@ const logger = require('../../commons/logger').getLogger('pixel-validation-step-
 
 class PixelValidationStepAction extends StepAction {
     async performAction() {
-        const { shouldUseVisualGrid, applitoolsSdkConfig: config } = this.context;
+        const { shouldUseVisualGrid, applitoolsSdkConfig: config, testResultId } = this.context;
         this.runContext = this.context.getRunContext(undefined);
-        const eyeManager = await eyeSdkService.getManager(shouldUseVisualGrid, this.context.config.applitoolsConcurrency || 5);
+        const eyeManager = await eyeSdkService.getManager(shouldUseVisualGrid, this.context.config.applitoolsConcurrency || 5, testResultId);
         const targetElementData = this.getTarget() || {};
         try {
             const openedEye = await eyeManager.openEyes({ driver: this.driver.client, config });

package/player/stepActions/salesforceAutoLoginStepAction.js

@@ -7,7 +7,7 @@ class SalesforceAutoLoginStepAction extends NavigationStepAction {
         try {
             salesforceUrl = await this.updateBaseUrl(salesforceUrl);
             await this.driver.url(salesforceUrl);
-            await Promise.delay(1000); // wait a little for the page to load (fixes screenshots and clicking on elements in username verification screen)
+            await Promise.delay(5000); // wait a little for the page to load (fixes screenshots and clicking on elements in username verification screen)
             let newUrl = await this.driver.getUrl();
             // Verify username screen
             const isUsernameVerificationNeeded = newUrl.includes(this.step.USERNAME_VERIFICATION_PATH_ID);
@@ -19,10 +19,10 @@ class SalesforceAutoLoginStepAction extends NavigationStepAction {
                     var done = arguments[1];
                     done();
                 `, timeout);
-                await Promise.delay(1500);
+                await Promise.delay(5000);
                 newUrl = await this.driver.getUrl(); // If we managed to continue correctly we want to get the new redirected url
             }
-            await Promise.delay(700);
+            await Promise.delay(1500);
             return {
                 success: true,
                 newUrl,

package/player/stepActions/stepActionRegistrar.js

@@ -21,9 +21,10 @@ const RefreshStepAction = require('./RefreshStepAction');
 const ApiStepAction = require('./apiStepAction');
 const ExtractTextStepAction = require('./extractTextStepAction');
 const TdkHybridStepAction = require('./tdkHybridStepAction');
-const SalesforceAutoLoginStepAction = require('./salesforceAutoLoginStepAction');
 const PixelValidationStepAction = require('./pixelValidationStepAction');
 
+const SalesforceAutoLoginStepAction = require('./salesforceAutoLoginStepAction');
+
 const CliJsStepAction = require('./cliJsStepAction');
 const CliConditionStepAction = require('./cliConditionStepAction');
 const NodePackageStepAction = require('./nodePackageStepAction');

package/player/utils/eyeSdkService.js

@@ -1,5 +1,9 @@
 // https://github.com/applitools/eyes.sdk.javascript1/blob/master/packages/eyes-webdriverio-4/src/spec-driver.ts
 
+/**
+ * @type {{ EyeSdkBuilder: typeof import('../../../../clickim/src/background/eyeSdkBuilder').EyeSdkBuilder }}
+ */
+const sessionPlayer = require('../../commons/getSessionPlayerRequire');
 const { makeSDK } = require('@applitools/eyes-sdk-core');
 const { W3C_ELEMENT_ID } = require('../constants');
 const _ = require('lodash');
@@ -222,8 +226,10 @@ class EyeSdkService {
             VisualGridClient: require('@applitools/visual-grid-client'),
         });
     }
-    getManager(useVisualGrid, concurrency) {
-        return this.sdk.makeManager({ type: useVisualGrid ? 'vg' : 'classic', concurrency });
+    async getManager(useVisualGrid, concurrency, batchId) {
+        const manager = await this.sdk.makeManager({ type: useVisualGrid ? 'vg' : 'classic', concurrency });
+        sessionPlayer.EyeSdkBuilder.rememberCreatedBatch(batchId);
+        return manager;
     }
 }
 

package/reports/consoleReporter.js

@@ -7,6 +7,8 @@ const constants = require('../commons/constants');
 const featureAvailabilityService = require('../commons/featureAvailabilityService');
 const { getAbortedTests, getFailedTests, getPassedTests, getFailureEvaluatingCount, getSkippedCount } = require('./reporterUtils');
 
+const colorize = { success: chalk.green, warn: chalk.yellow, error: chalk.red };
+
 class ConsoleReporter {
     constructor(options, branchToUse) {
         this.options = options;
@@ -16,12 +18,8 @@ class ConsoleReporter {
         this.branchToUse = branchToUse;
     }
 
-    printWorkerMessage(workerId, message) {
-        if (this.config.showWorkerNames) {
-            console.log(`W:${workerId} ${message}`);
-        } else {
-            console.log(message);
-        }
+    toWorkerIdPrefix(workerId) {
+        return this.config.showWorkerNames ? `W:${workerId}` : '';
     }
 
     printWorkerDivider() {
@@ -32,7 +30,7 @@ class ConsoleReporter {
         const type = isCodeMode ? 'File' : 'Test';
         const testIdLabel = test.isTestsContainer ? '' : `(${test.testId})`;
         const testUrlLabel = test.isTestsContainer ? '' : `url: ${chalk.underline(utils.getTestUrl(this.options.editorUrl, this.options.project, test.testId, test.resultId, this.branchToUse))}`;
-        this.printWorkerMessage(workerId, `${type} "${test.name}" started ${testIdLabel} ${testUrlLabel}`.trim());
+        console.log(this.toWorkerIdPrefix(workerId), `${type} "${test.name}" started ${testIdLabel} ${testUrlLabel}`.trim());
     }
 
     onTestFinished(test, workerId, isRerun, isCodeMode) {
@@ -42,7 +40,9 @@ class ConsoleReporter {
         }
         const testStatus = test.success ? constants.runnerTestStatus.PASSED : constants.runnerTestStatus.FAILED;
         const testIdLabel = test.isTestsContainer ? ' ' : `(${test.testId})`;
-        this.printWorkerMessage(workerId, `Test "${test.name}" finished status: ${testStatus} ${testIdLabel} duration: ${utils.getDuration(test.duration)}`);
+        const color = colorize[test.success ? 'success' : 'error'];
+
+        console.log(color(this.toWorkerIdPrefix(workerId), `Test "${test.name}" finished status: ${testStatus} ${testIdLabel} duration: ${utils.getDuration(test.duration)}`));
     }
 
     printAllFailedTests(failedTests) {
@@ -54,8 +54,8 @@ class ConsoleReporter {
                 }
                 return `${failedTest.name} : ${testUrl}`;
             });
-            console.log('Failed runs are:');
-            console.log(failedTestStrings.join('\n\r'));
+            console.log(colorize.error('Failed runs are:'));
+            console.log(colorize.error(failedTestStrings.join('\n\r')));
         }
     }
 
@@ -79,15 +79,20 @@ class ConsoleReporter {
         }
 
         const planName = this.buildTestPlanName(isAnonymous, testPlanName, isCodeMode);
+
+        let message;
+        const color = colorize[failed ? 'error' : 'success'];
+
         if (isCodeMode || planName.trim() === '' || planName.trim() === 'Anonymous') {
-            this.printWorkerDivider();
-            console.log(`Tests completed. PASSED: ${passed} FAILED: ${failed}${failedEvaluatingString} ABORTED: ${aborted}${skippedString} Duration: ${utils.getDuration(duration)} (Execution ID: ${executionId})`);
-            this.printWorkerDivider();
+            message = `Tests completed. PASSED: ${passed} FAILED: ${failed}${failedEvaluatingString} ABORTED: ${aborted}${skippedString} Duration: ${utils.getDuration(duration)} (Execution ID: ${executionId})`;
         } else {
-            this.printWorkerDivider();
-            console.log(`Test plan${planName} completed PASSED: ${passed} FAILED: ${failed}${failedEvaluatingString} ABORTED: ${aborted}${skippedString} Duration: ${utils.getDuration(duration)} (${executionId})`);
-            this.printWorkerDivider();
+            message = `Test plan${planName} completed PASSED: ${passed} FAILED: ${failed}${failedEvaluatingString} ABORTED: ${aborted}${skippedString} Duration: ${utils.getDuration(duration)} (${executionId})`;
         }
+
+        this.printWorkerDivider();
+        console.log(color(message));
+        this.printWorkerDivider();
+
         this.printAllFailedTests(failedTests);
     }
 
@@ -138,24 +143,24 @@ class ConsoleReporter {
     onGetSlot(workerId, browser) {
         const gridNameOrId = this.options.grid || this.options.gridId;
         if (gridNameOrId) {
-            this.printWorkerMessage(workerId, `Get ${chalk.underline(browser)} slot from ${chalk.underline(gridNameOrId)}`);
+            console.log(this.toWorkerIdPrefix(workerId), `Get ${chalk.underline(browser)} slot from ${chalk.underline(gridNameOrId)}`);
         }
     }
 
     onGetSession(workerId, testName, mode) {
-        this.printWorkerMessage(workerId, `Get browser to run ${chalk.underline(testName)}`);
+        console.log(this.toWorkerIdPrefix(workerId), `Get browser to run ${chalk.underline(testName)}`);
     }
 
     onWaitToTestStart(workerId) {
-        this.printWorkerMessage(workerId, 'Wait for test start');
+        console.log(this.toWorkerIdPrefix(workerId), 'Wait for test start');
     }
 
     onWaitToTestComplete(workerId, isCodeMode, debuggerAddress) {
         const type = isCodeMode ? 'file' : 'test';
-        this.printWorkerMessage(workerId, `Wait for ${type} complete`);
+        console.log(this.toWorkerIdPrefix(workerId), `Wait for ${type} complete`);
         if (debuggerAddress && isCodeMode) {
             // TODO(Benji) decide with Amitai what we want to do with this
-            this.printWorkerMessage(workerId, `Chrome Debugger available at ${debuggerAddress}`);
+            console.log(this.toWorkerIdPrefix(workerId), `Chrome Debugger available at ${debuggerAddress}`);
         }
     }
 
@@ -166,11 +171,11 @@ class ConsoleReporter {
         // heuristic, show the message on the same attempt
         const gridNameOrId = this.options.grid || this.options.gridId;
         if (gridNameOrId) { // if the user passes a grid or a gridId - show those
-            this.printWorkerMessage(workerId, `It is taking us some time to get a browser from the grid ${gridNameOrId}`);
+            console.log(colorize.warn(this.toWorkerIdPrefix(workerId), `It is taking us some time to get a browser from the grid ${gridNameOrId}`));
         } else if (this.options.usingLocalChromeDriver) {
-            this.printWorkerMessage(workerId, 'We are having issues starting ChromeDriver for you locally');
+            console.log(colorize.warn(this.toWorkerIdPrefix(workerId), 'We are having issues starting ChromeDriver for you locally'));
         } else if (this.options.host) {
-            this.printWorkerMessage(workerId, `We are having issues reaching your Selenium grid at ${this.options.host}:${this.options.port || 4444}`);
+            console.log(colorize.warn(this.toWorkerIdPrefix(workerId), `We are having issues reaching your Selenium grid at ${this.options.host}:${this.options.port || 4444}`));
         } else {
             // in other cases - print nothing
         }

package/reports/junitReporter.js

@@ -50,9 +50,6 @@ class JunitReporter {
 }
 
 function getPrintName(testResult) {
-    if (!featureFlags.flags.testNameTestDataInJunitReport.isEnabled()) {
-        return testResult.name;
-    }
     const testData = testResult.testData || {};
     const testDataNumber = typeof testData.total === 'number' ? ` - ${testData.index} / ${testData.total} Data set` : '';
     return `${testResult.name}${testDataNumber}`;

package/runner.js

@@ -286,16 +286,7 @@ async function init(options) {
     reporter.setOptions(options, branchToUse);
 }
 
-function run(options, customExtensionLocalLocation) {
-    perf.log('in runner.js run');
-
-    if (options.files.length > 0 && !featureFlags.flags.enableTDKRun.isEnabled()) {
-        throw new ArgError('run command is not supported, please sign up to the Testim Development Kit beta - https://go.testim.io/testims-new-beta-testim-tdk-testim-development-kit-testautomation');
-    }
-    return runRunner(options, customExtensionLocalLocation);
-}
-
 module.exports = {
-    run,
+    run: runRunner,
     init: Promise.method(init),
 };

package/runners/ParallelWorkerManager.js

@@ -46,14 +46,14 @@ class ParallelWorkerManager {
         return Array.from(new Array(count), createWorker);
     }
 
-    async runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount, stopOnError) {
+    async runTests(testList, testStatus, executionId, executionName, options, branchToUse, authData, workerCount, stopOnError) {
         if (testList && testList.length === 0) {
             return undefined;
         }
 
         const runAndWaitToComplete = token => new Promise((resolve, reject) => {
             const projectId = options.project;
-            const executionQueue = new ExecutionQueue(executionId, testList, options, branchToUse, testStatus);
+            const executionQueue = new ExecutionQueue(executionId, executionName, testList, options, branchToUse, testStatus);
 
             const combinedTestResults = {};
             const testCount = testList.length;

package/runners/TestPlanRunner.js

@@ -29,30 +29,30 @@ class TestPlanRunner {
         this.workerManager = new ParallelWorkerManager(customExtensionLocalLocation);
         this.startTime = Date.now();
     }
-    runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, testStatus) {
+    runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, executionName, testStatus) {
         const executionResults = {};
         const authData = testimCustomToken.getTokenV3UserData();
 
-        const runBeforeTests = (beforeTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runBeforeTests = (beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = 1;
             const stopOnError = true;
-            return this.workerManager.runTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(beforeTestsResults => Object.assign(executionResults, beforeTestsResults));
         };
 
-        const runTestPlanTests = (tests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runTestPlanTests = (tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
             const stopOnError = false;
             perf.log('right before this.workerManager.runTests');
-            return this.workerManager.runTests(tests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .log('right after this.workerManager.runTests')
                 .then(testsResults => Object.assign(executionResults, testsResults));
         };
 
-        const runAfterTests = (afterTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runAfterTests = (afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = 1;
             const stopOnError = false;
-            return this.workerManager.runTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(afterTestsResults => Object.assign(executionResults, afterTestsResults));
         };
 
@@ -63,13 +63,19 @@ class TestPlanRunner {
         const sessionType = utils.getSessionType(tpOptions);
         analyticsService.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
         perf.log('right before runBeforeTests');
-        return runBeforeTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData)
+        return runBeforeTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData)
             .log('right before runTestPlanTests')
-            .then(() => runTestPlanTests(tests, testStatus, executionId, tpOptions, branchToUse, authData))
+            .then(() => runTestPlanTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
             .log('right after runTestPlanTests')
-            .then(() => runAfterTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData))
+            .then(() => runAfterTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
             .then(() => executionResults)
-            .catch(StopRunOnError, () => catchBeforeTestsFailed(executionId));
+            .catch(err => {
+                logger.error('error running test plan', { err });
+                if (err instanceof StopRunOnError) {
+                    return catchBeforeTestsFailed(executionId);
+                }
+                throw err;
+            });
     }
 
     async initRealDataService(projectId) {
@@ -144,10 +150,6 @@ class TestPlanRunner {
 
         const isCodeMode = tpOptions.files.length > 0;
 
-        if (isCodeMode && tpOptions.mode === constants.CLI_MODE.SELENIUM) {
-            // in selenium mode we don't need to wait for the runner and clickim to sync, so we don't need to wait for reports.
-            testStatus.setAsyncReporting(true);
-        }
         const testListInfoPromise = tpOptions.lightweightMode && tpOptions.lightweightMode.onlyTestIdsNoSuite ?
             { beforeTests, tests, afterTests } :
             testStatus.executionStart(executionId, projectId, this.startTime, testPlanName);
@@ -165,7 +167,7 @@ class TestPlanRunner {
         }
 
         perf.log('before runTestAllPhases');
-        const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testStatus);
+        const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testPlanName || 'All Tests', testStatus);
         const childResults = await Bluebird.resolve(childTestResults)
             .timeout(TDK_CHILD_RESULTS_TIMEOUT)
             .catch(async () => {

package/services/gridService.js

@@ -245,10 +245,6 @@ async function getTestPlanGridData(options) {
     if (testPlanGrids.includes(undefined)) {
         throw new ArgError('failed to find one of the test plan defined grid');
     }
-    const gridTypes = _(testPlanGrids).map(grid => grid.type).uniq().value();
-    if (gridTypes.includes('testimMobile') && gridTypes.length > 1) {
-        throw new ArgError('Test plans cannot include two different grid types');
-    }
     return getSerializableObject(_.first(testPlanGrids));
 }
 

package/stepPlayers/hybridStepPlayback.js

@@ -6,6 +6,7 @@ const perfLogger = require('../commons/performance-logger');
 const MemoryFS = require('memory-fs');
 const mfs = new MemoryFS();
 const AbortController = require("abort-controller");
+const logger = require('../commons/logger').getLogger('hybrid-step-playback');
 
 /**
  * @type {Map<string, import("abort-controller")>}
@@ -119,10 +120,12 @@ module.exports.execute = async function execute(step, context, driver, loginData
         }
 
         return { success: false, shouldRetry: false, reason: 'unknown hybrid format ' + hybridFunction.type };
+    } catch (err) {
+        logger.log('error running hybrid step', { err });
     } finally {
         runningStepsAbortControllersRegistry.delete(context.stepResultId);
     }
-}
+};
 
 module.exports.abort = function abort(stepResultId) {
     const abortController = runningStepsAbortControllersRegistry.get(stepResultId);

package/stepPlayers/tdkHybridStepPlayback.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const _ = require('lodash');
 const sessionPlayerInit = require('../commons/getSessionPlayerRequire');
 const perfLogger = require('../commons/performance-logger');
 const { guid } = require('../utils');

package/testRunHandler.js

@@ -7,6 +7,7 @@ const testimServicesApi = require('./commons/testimServicesApi');
 const { timeoutMessages, CLI_MODE } = require('./commons/constants');
 const logger = require('./commons/logger').getLogger('test-run-handler');
 const perf = require('./commons/performance-logger');
+const { URL } = require('url');
 const Promise = require('bluebird');
 const _ = require('lodash');
 const remoteStepPlayback = require('./stepPlayers/remoteStepPlayback');
@@ -14,13 +15,13 @@ const utils = require('./utils');
 const config = require('./commons/config');
 const analytics = require('./commons/testimAnalytics');
 const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
-const featureFlags = require('./commons/featureFlags');
 const { preloadTests } = require('./commons/preloadTests');
 
 const RETRIES_ON_TIMEOUT = 3;
 
-const TestRun = function (executionId, test, options, branchToUse, testRunStatus) {
+const TestRun = function (executionId, executionName, test, options, branchToUse, testRunStatus) {
     this._executionId = executionId;
+    this._executionName = executionName;
     this._testStatus = test.testStatus;
     this._testId = test.testId;
     this._testName = test.name;
@@ -80,6 +81,10 @@ TestRun.prototype.getExecutionId = function () {
     return this._executionId;
 };
 
+TestRun.prototype.getExecutionName = function () {
+    return this._executionName;
+};
+
 TestRun.prototype.getNativeAppData = function () {
     if (this._nativeApp && !this._options.baseUrl) {
         return this._nativeApp;
@@ -121,6 +126,7 @@ TestRun.prototype.getRunRequestParams = async function () {
         refreshToken: testimCustomToken.getRefreshToken(),
         projectId: this._options.project,
         executionId: this._executionId,
+        executionName: this._executionName,
         testId: this._testId,
         resultId: this._testResultId,
         baseUrl: this._baseUrl,
@@ -251,9 +257,6 @@ TestRun.prototype.clearTestResult = function () {
                 testRetryKey: this.getRetryKey(),
             });
         });
-    if (this._testRunStatus.asyncReporting) {
-        return Promise.resolve();
-    }
     return this.clearTestResultFinished;
 };
 
@@ -316,6 +319,26 @@ TestRun.prototype.isRetryKeyMismatch = function (testResult) {
     return testResult.testRetryKey && (testResult.testRetryKey !== this.getRetryKey());
 };
 
+TestRun.prototype.validateRunConfig = function () {
+    const baseUrl = this.getBaseUrl();
+    const { browserValue } = this.getRunConfig();
+
+    if (baseUrl && browserValue === 'safari') {
+        let parsedUrl;
+        try {
+            parsedUrl = new URL(baseUrl);
+        } catch (err) {
+            // ignore invalid URLs (missing http:// or https:// prefix)
+            return;
+        }
+        const { username, password } = parsedUrl;
+
+        if (username || password) {
+            throw new Error('Basic authentication in URL is not supported in Safari');
+        }
+    }
+};
+
 TestRun.prototype.onStarted = function (startTimeout) {
     return new Promise(resolve => {
         // We can't leave the test result as it may remove other listeners as well
@@ -467,7 +490,7 @@ TestRun.prototype.onCompleted = function () {
                             waitForTestEnd();
                         }
                     } catch (err) {
-                        logger.error('failed to check is complete', {err});
+                        logger.error('failed to check is complete', { err });
                         waitForTestEnd();
                     }
                 }, 3000);
@@ -510,7 +533,7 @@ TestRun.prototype.isAllowReportTestResultRetries = function () {
 TestRun.prototype.onRetry = async function () {
     this._previousTestResultId = this._testResultId;
 
-    if (!featureFlags.flags.countRetries.isEnabled() || !this.isAllowReportTestResultRetries()) {
+    if (!this.isAllowReportTestResultRetries()) {
         return;
     }
 

package/testRunStatus.js

@@ -23,11 +23,11 @@ const gitRepoUrl = process.env.GIT_URL || process.env.CIRCLE_REPOSITORY_URL;
 const runnerVersion = utils.getRunnerVersion();
 
 
-function runHook(fn, object) {
+function runHook(fn, ...args) {
     if (!fn || typeof fn !== 'function') {
         return Promise.resolve();
     }
-    return Promise.try(() => fn(object) || {}).catch(err => {
+    return Promise.try(() => fn(...args) || {}).catch(err => {
         logger.warn('failed to run hook', { err });
         throw new ArgError(`failed to run hook promise ${err.message}`);
     });
@@ -43,7 +43,6 @@ const RunStatus = function (testInfoList, options, testPlanId, branchToUse) {
     this.exportsGlobal = {};
     this.testInfoList = testInfoList;
 
-    this.asyncReporting = false; // whether or not we wait for result reporting
     this.executionStartedPromise = Promise.resolve();
 
     const browserNames = utils.getUniqBrowsers(options, testInfoList);
@@ -92,9 +91,6 @@ const RunStatus = function (testInfoList, options, testPlanId, branchToUse) {
 RunStatus.prototype.waitForExecutionStartedFinished = function () {
     return this.executionStartedPromise;
 };
-RunStatus.prototype.setAsyncReporting = function (isAsyncReportingEnabled = false) {
-    this.asyncReporting = isAsyncReportingEnabled;
-};
 RunStatus.prototype.getTestResult = function (resultId) {
     return this.testRunStatus[resultId];
 };
@@ -157,7 +153,7 @@ RunStatus.prototype.updateTestStatusRunning = function (test, executionId, testR
         return this.executionStartedPromise;
     }
 
-    const res = servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults)
+    return servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults)
         .catch(err => {
             logger.error('failed to upload test data artifact (runner)', { err });
             return '';
@@ -169,19 +165,13 @@ RunStatus.prototype.updateTestStatusRunning = function (test, executionId, testR
             await this.executionStartedPromise;
             return servicesApi.updateTestStatus(projectId, executionId, test.testId, test.resultId, 'RUNNING', { startTime: test.startTime, config: testConfig, remoteRunId, testRetryKey });
         });
-    if (this.asyncReporting) {
-        // silence is golden
-    } else {
-        return res;
-    }
-    return servicesApi.updateTestStatus(projectId, executionId, test.testId, test.resultId, 'RUNNING', { startTime: test.startTime, config: test.config, remoteRunId, testRetryKey });
 };
 
 RunStatus.prototype.testStartReport = function (test, executionId, testRetryKey) {
     if (utils.isQuarantineAndNotRemoteRun(test, this.options)) {
         return Promise.resolve();
     }
-    return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }))
+    return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }), this.options.userData.loginData.token)
         .then(params => {
             this.options.runParams[test.resultId] = test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
             test.startTime = Date.now();
@@ -278,7 +268,7 @@ RunStatus.prototype.testEndReport = async function (test, executionId, result, t
     const globalParameters = result.exportsGlobal;
     try {
         try {
-            await runHook(this.options.afterTest, Object.assign({}, test, { globalParameters }));
+            await runHook(this.options.afterTest, Object.assign({}, test, { globalParameters }), this.options.userData.loginData.token);
         } catch (err) {
             logger.error('HOOK threw an error', { test: test.testId, err });
             // eslint-disable-next-line no-console
@@ -406,10 +396,7 @@ RunStatus.prototype.executionStart = function (executionId, projectId, startTime
                     const ret = servicesApi.reportExecutionStarted(data);
                     this.executionStartedPromise = ret;
                     ret.catch(e => logger.error(e));
-                    if (!this.asyncReporting) {
-                        return ret;
-                    }
-                    return undefined;
+                    return ret;
                 });
             };
 

package/workers/BaseWorker.js

@@ -387,6 +387,7 @@ class BaseWorker {
             !disableRemoteStep && remoteStepService.joinToRemoteStep(this.testResultId),
             !disableResults && testResultService.joinToTestResult(this.testResultId, this.testId),
         ])
+            .then(() => testRunHandler.validateRunConfig())
             .then(() => this.runTest(testRunHandler, this.customExtensionLocalLocation, shouldRerun))
             .then(testResult => onRunComplete(testResult, testRunHandler))
             .then(result => {

package/workers/WorkerSelenium.js

@@ -88,6 +88,9 @@ class WorkerSelenium extends BaseWorker {
 
         setupCliPerformanceMonitoring(sessionPlayer);
 
+        sessionPlayer.playbackManager.executionId = testRunHandler.getExecutionId();
+        sessionPlayer.playbackManager.executionName = testRunHandler.getExecutionName();
+
         sessionPlayer.setLightweightMode(this.options.lightweightMode);
         if (sessionPlayerInit.localAssetService) {
             sessionPlayerInit.localAssetService.initialize({ serverUrl: this.options.localRCASaver });