@testim/testim-cli 3.196.0 → 3.200.0

package/services/gridService.test.js

@@ -175,19 +175,19 @@ describe('gridService', () => {
 
     describe('getGridSlot', () => {
         let getGridByIdStub;
-        let getGridByProjectStub;
+        let getGridByNameStub;
         let onGridSlot;
         let addItemToGridCacheStub;
 
         beforeEach(() => {
             getGridByIdStub = sinon.stub(servicesApi, 'getGridById').resolves({ grid: { gridId: 'gridId', type: 'gridId' }, status: 'success' });
-            getGridByProjectStub = sinon.stub(servicesApi, 'getGridByProject').resolves({ grid: { gridId: 'gridId', type: 'gridName' }, status: 'success' });
+            getGridByNameStub = sinon.stub(servicesApi, 'getGridByName').resolves({ grid: { gridId: 'gridId', type: 'gridName' }, status: 'success' });
             addItemToGridCacheStub = sinon.stub(gridService, 'addItemToGridCache').callThrough();
             onGridSlot = sinon.stub().resolves();
         });
         afterEach(() => {
             getGridByIdStub.restore();
-            getGridByProjectStub.restore();
+            getGridByNameStub.restore();
             addItemToGridCacheStub.restore();
         });
 
@@ -195,7 +195,7 @@ describe('gridService', () => {
             const slot = await gridService.getGridSlot('browser', 'executionId', 'testResultId', onGridSlot, { useLocalChromeDriver: true }, 'workerId');
             expect(slot).to.eql({ mode: 'local' });
             sinon.assert.notCalled(getGridByIdStub);
-            sinon.assert.notCalled(getGridByProjectStub);
+            sinon.assert.notCalled(getGridByNameStub);
             sinon.assert.notCalled(addItemToGridCacheStub);
             sinon.assert.calledOnce(onGridSlot);
         });
@@ -204,7 +204,7 @@ describe('gridService', () => {
             const slot = await gridService.getGridSlot('browser', 'executionId', 'testResultId', onGridSlot, { useChromeLauncher: true }, 'workerId');
             expect(slot).to.eql({ mode: 'local' });
             sinon.assert.notCalled(getGridByIdStub);
-            sinon.assert.notCalled(getGridByProjectStub);
+            sinon.assert.notCalled(getGridByNameStub);
             sinon.assert.notCalled(addItemToGridCacheStub);
             sinon.assert.calledOnce(onGridSlot);
         });
@@ -213,7 +213,7 @@ describe('gridService', () => {
             const slot = await gridService.getGridSlot('browser', 'executionId', 'testResultId', onGridSlot, { host: 'localhost', port: 4444 }, 'workerId');
             expect(slot).to.shallowDeepEqual({ type: 'hostAndPort', host: 'localhost', port: 4444 });
             sinon.assert.notCalled(getGridByIdStub);
-            sinon.assert.notCalled(getGridByProjectStub);
+            sinon.assert.notCalled(getGridByNameStub);
             sinon.assert.notCalled(addItemToGridCacheStub);
             sinon.assert.calledOnce(onGridSlot);
         });
@@ -222,7 +222,7 @@ describe('gridService', () => {
             const slot = await gridService.getGridSlot('browser', 'executionId', 'testResultId', onGridSlot, { gridId: 'gridId' }, 'workerId');
             expect(slot).to.shallowDeepEqual({ type: 'gridId', gridId: 'gridId' });
             sinon.assert.calledOnce(getGridByIdStub);
-            sinon.assert.notCalled(getGridByProjectStub);
+            sinon.assert.notCalled(getGridByNameStub);
             sinon.assert.calledOnce(addItemToGridCacheStub);
             sinon.assert.calledOnce(onGridSlot);
         });
@@ -230,7 +230,7 @@ describe('gridService', () => {
         it('should get grid from server when passing grid name', async () => {
             const slot = await gridService.getGridSlot('browser', 'executionId', 'testResultId', onGridSlot, { grid: 'gridName' }, 'workerId');
             expect(slot).to.shallowDeepEqual({ type: 'gridName', gridId: 'gridId' });
-            sinon.assert.calledOnce(getGridByProjectStub);
+            sinon.assert.calledOnce(getGridByNameStub);
             sinon.assert.notCalled(getGridByIdStub);
             sinon.assert.calledOnce(addItemToGridCacheStub);
             sinon.assert.calledOnce(onGridSlot);
@@ -282,9 +282,9 @@ describe('gridService', () => {
         });
 
         it('should call releaseGridSlot with the correct parameters', async () => {
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
             await gridService.releaseGridSlot('workerId', 'projectId');
-            sinon.assert.calledWith(releaseGridSlotStub, 'projectId', 'slotId', 'gridId', 'chrome');
+            sinon.assert.calledWith(releaseGridSlotStub, 'companyId', 'projectId', 'slotId', 'gridId', 'chrome');
         });
 
         it('should not call releaseGridSlot if the workerId is not in the cache', async () => {
@@ -293,16 +293,16 @@ describe('gridService', () => {
         });
 
         it('should not call releaseGridSlot if no slotId', async () => {
-            gridService.addItemToGridCache('workerId', 'gridId');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId');
             await gridService.releaseGridSlot('workerId', 'projectId');
             sinon.assert.notCalled(releaseGridSlotStub);
         });
 
         it('should handle releaseGridSlot request error', async () => {
             releaseGridSlotStub.rejects({ });
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
             await gridService.releaseGridSlot('workerId', 'projectId');
-            sinon.assert.calledWith(releaseGridSlotStub, 'projectId', 'slotId', 'gridId', 'chrome');
+            sinon.assert.calledWith(releaseGridSlotStub, 'companyId', 'projectId', 'slotId', 'gridId', 'chrome');
         });
     });
 
@@ -322,15 +322,15 @@ describe('gridService', () => {
         });
 
         it('should send keepAlive to server on an interval', async () => {
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
             await gridService.keepAlive.start('projectId');
             clock.tick(10010);
-            sinon.assert.calledOnceWithExactly(keepAliveStub, 'projectId', [{ gridId: 'gridId', slotId: 'slotId', browser: 'chrome' }]);
+            sinon.assert.calledOnceWithExactly(keepAliveStub, 'projectId', [{ gridId: 'gridId', companyId: 'companyId', slotId: 'slotId', browser: 'chrome' }]);
             await gridService.releaseGridSlot('workerId', 'projectId');
         });
 
         it('should send keepAlive to server every 10 seconds', async () => {
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
             await gridService.keepAlive.start('projectId');
             clock.tick(30010);
             sinon.assert.calledThrice(keepAliveStub);
@@ -345,7 +345,7 @@ describe('gridService', () => {
 
         it('should handle keepAlive request error', async () => {
             keepAliveStub.rejects({ });
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
             await gridService.keepAlive.start('projectId');
             clock.tick(10010);
             sinon.assert.calledOnce(keepAliveStub);
@@ -353,13 +353,13 @@ describe('gridService', () => {
         });
 
         it('should release all slots when ending', async () => {
-            gridService.addItemToGridCache('workerId', 'gridId', 'slotId', 'chrome');
-            gridService.addItemToGridCache('workerId1', 'gridId', 'slotId1', 'firefox');
+            gridService.addItemToGridCache('workerId', 'companyId', 'gridId', 'slotId', 'chrome');
+            gridService.addItemToGridCache('workerId1', 'companyId', 'gridId', 'slotId1', 'firefox');
             await gridService.keepAlive.start('projectId');
             await gridService.keepAlive.end('projectId');
             sinon.assert.calledTwice(releaseGridSlotStub);
-            sinon.assert.calledWith(releaseGridSlotStub, 'projectId', 'slotId', 'gridId', 'chrome');
-            sinon.assert.calledWith(releaseGridSlotStub, 'projectId', 'slotId1', 'gridId', 'firefox');
+            sinon.assert.calledWith(releaseGridSlotStub, 'companyId', 'projectId', 'slotId', 'gridId', 'chrome');
+            sinon.assert.calledWith(releaseGridSlotStub, 'companyId', 'projectId', 'slotId1', 'gridId', 'firefox');
         });
 
         it('should not release slots if there is no slots in use', async () => {

package/stepPlayers/hybridStepPlayback.js

@@ -6,6 +6,7 @@ const perfLogger = require('../commons/performance-logger');
 const MemoryFS = require('memory-fs');
 const mfs = new MemoryFS();
 const AbortController = require("abort-controller");
+const logger = require('../commons/logger').getLogger('hybrid-step-playback');
 
 /**
  * @type {Map<string, import("abort-controller")>}
@@ -119,10 +120,12 @@ module.exports.execute = async function execute(step, context, driver, loginData
         }
 
         return { success: false, shouldRetry: false, reason: 'unknown hybrid format ' + hybridFunction.type };
+    } catch (err) {
+        logger.log('error running hybrid step', { err });
     } finally {
         runningStepsAbortControllersRegistry.delete(context.stepResultId);
     }
-}
+};
 
 module.exports.abort = function abort(stepResultId) {
     const abortController = runningStepsAbortControllersRegistry.get(stepResultId);

package/stepPlayers/tdkHybridStepPlayback.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const _ = require('lodash');
 const sessionPlayerInit = require('../commons/getSessionPlayerRequire');
 const perfLogger = require('../commons/performance-logger');
 const { guid } = require('../utils');

package/testRunHandler.js

@@ -7,6 +7,7 @@ const testimServicesApi = require('./commons/testimServicesApi');
 const { timeoutMessages, CLI_MODE } = require('./commons/constants');
 const logger = require('./commons/logger').getLogger('test-run-handler');
 const perf = require('./commons/performance-logger');
+const { URL } = require('url');
 const Promise = require('bluebird');
 const _ = require('lodash');
 const remoteStepPlayback = require('./stepPlayers/remoteStepPlayback');
@@ -241,7 +242,7 @@ TestRun.prototype.clearTestResult = function () {
             // make sure the execution is created by now
             await this._testRunStatus.waitForExecutionStartedFinished();
             // we probably can save this backend call by initializing the execution
-            return testResultService.clearTestResult(this._options.project, this._testResultId, this._testId, {
+            return testimServicesApi.clearTestResult(this._options.project, this._testResultId, this._testId, {
                 name: this._testName,
                 resultId: this._testResultId,
                 status: 'pending',
@@ -251,9 +252,6 @@ TestRun.prototype.clearTestResult = function () {
                 testRetryKey: this.getRetryKey(),
             });
         });
-    if (this._testRunStatus.asyncReporting) {
-        return Promise.resolve();
-    }
     return this.clearTestResultFinished;
 };
 
@@ -316,6 +314,26 @@ TestRun.prototype.isRetryKeyMismatch = function (testResult) {
     return testResult.testRetryKey && (testResult.testRetryKey !== this.getRetryKey());
 };
 
+TestRun.prototype.validateRunConfig = function () {
+    const baseUrl = this.getBaseUrl();
+    const { browserValue } = this.getRunConfig();
+
+    if (baseUrl && browserValue === 'safari') {
+        let parsedUrl;
+        try {
+            parsedUrl = new URL(baseUrl);
+        } catch (err) {
+            // ignore invalid URLs (missing http:// or https:// prefix)
+            return;
+        }
+        const { username, password } = parsedUrl;
+
+        if (username || password) {
+            throw new Error('Basic authentication in URL is not supported in Safari');
+        }
+    }
+};
+
 TestRun.prototype.onStarted = function (startTimeout) {
     return new Promise(resolve => {
         // We can't leave the test result as it may remove other listeners as well
@@ -467,7 +485,7 @@ TestRun.prototype.onCompleted = function () {
                             waitForTestEnd();
                         }
                     } catch (err) {
-                        logger.error('failed to check is complete', {err});
+                        logger.error('failed to check is complete', { err });
                         waitForTestEnd();
                     }
                 }, 3000);

package/testRunStatus.js

@@ -23,11 +23,11 @@ const gitRepoUrl = process.env.GIT_URL || process.env.CIRCLE_REPOSITORY_URL;
 const runnerVersion = utils.getRunnerVersion();
 
 
-function runHook(fn, object) {
+function runHook(fn, ...args) {
     if (!fn || typeof fn !== 'function') {
         return Promise.resolve();
     }
-    return Promise.try(() => fn(object) || {}).catch(err => {
+    return Promise.try(() => fn(...args) || {}).catch(err => {
         logger.warn('failed to run hook', { err });
         throw new ArgError(`failed to run hook promise ${err.message}`);
     });
@@ -43,7 +43,6 @@ const RunStatus = function (testInfoList, options, testPlanId, branchToUse) {
     this.exportsGlobal = {};
     this.testInfoList = testInfoList;
 
-    this.asyncReporting = false; // whether or not we wait for result reporting
     this.executionStartedPromise = Promise.resolve();
 
     const browserNames = utils.getUniqBrowsers(options, testInfoList);
@@ -92,9 +91,6 @@ const RunStatus = function (testInfoList, options, testPlanId, branchToUse) {
 RunStatus.prototype.waitForExecutionStartedFinished = function () {
     return this.executionStartedPromise;
 };
-RunStatus.prototype.setAsyncReporting = function (isAsyncReportingEnabled = false) {
-    this.asyncReporting = isAsyncReportingEnabled;
-};
 RunStatus.prototype.getTestResult = function (resultId) {
     return this.testRunStatus[resultId];
 };
@@ -157,7 +153,7 @@ RunStatus.prototype.updateTestStatusRunning = function (test, executionId, testR
         return this.executionStartedPromise;
     }
 
-    const res = servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults)
+    return servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults)
         .catch(err => {
             logger.error('failed to upload test data artifact (runner)', { err });
             return '';
@@ -167,21 +163,15 @@ RunStatus.prototype.updateTestStatusRunning = function (test, executionId, testR
             delete testConfig.testData;
             testConfig.testDataUrl = testDataUrl;
             await this.executionStartedPromise;
-            return servicesApi.updateTestStatus('RUNNING', executionId, test.testId, test.resultId, test.startTime, null, null, null, testConfig, projectId, remoteRunId, testRetryKey);
+            return servicesApi.updateTestStatus(projectId, executionId, test.testId, test.resultId, 'RUNNING', { startTime: test.startTime, config: testConfig, remoteRunId, testRetryKey });
         });
-    if (this.asyncReporting) {
-        // silence is golden
-    } else {
-        return res;
-    }
-    return this.executionStartedPromise.then(() => servicesApi.updateTestStatus('RUNNING', executionId, test.testId, test.resultId, test.startTime, null, null, null, test.config, projectId, remoteRunId, testRetryKey));
 };
 
 RunStatus.prototype.testStartReport = function (test, executionId, testRetryKey) {
     if (utils.isQuarantineAndNotRemoteRun(test, this.options)) {
         return Promise.resolve();
     }
-    return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }))
+    return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }), this.options.userData.loginData.token)
         .then(params => {
             this.options.runParams[test.resultId] = test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
             test.startTime = Date.now();
@@ -201,9 +191,6 @@ RunStatus.prototype.onGridSlot = function (executionId, resultId, gridInfo) {
     const test = this.getTestResult(resultId);
     test.config.gridInfo = Object.assign({}, gridInfo, { key: undefined, user: undefined });
     logger.info('on get grid info', { gridInfo: test.config.gridInfo });
-    // we don't wait for this call since it's only for analytics/debugging sake.
-    const p = this.updateTestStatusRunning(test, executionId);
-    p.catch(() => {}); // suppress unhandled rejection
 };
 
 RunStatus.prototype.reportTestStatus = function (workerId, result, test, isRerun) {
@@ -277,11 +264,11 @@ RunStatus.prototype.testEnd = function (wid, result, executionId, sessionId, isR
     return test;
 };
 
-RunStatus.prototype.testEndReport = async function (test, executionId, result) {
+RunStatus.prototype.testEndReport = async function (test, executionId, result, testResultUpdates) {
     const globalParameters = result.exportsGlobal;
     try {
         try {
-            await runHook(this.options.afterTest, Object.assign({}, test, { globalParameters }));
+            await runHook(this.options.afterTest, Object.assign({}, test, { globalParameters }), this.options.userData.loginData.token);
         } catch (err) {
             logger.error('HOOK threw an error', { test: test.testId, err });
             // eslint-disable-next-line no-console
@@ -291,16 +278,23 @@ RunStatus.prototype.testEndReport = async function (test, executionId, result) {
             return undefined;
         }
 
-        return await servicesApi.updateTestStatus('FINISHED', executionId, test.testId, test.resultId, test.startTime, result.endTime, test.success, test.failureReason, null, this.options.project, this.options.remoteRunId, undefined, 5);
+        return await servicesApi.updateTestStatus(this.options.project, executionId, test.testId, test.resultId, 'FINISHED', {
+            startTime: test.startTime,
+            endTime: result.endTime,
+            success: test.success,
+            failureReason: test.failureReason,
+            remoteRunId: this.options.remoteRunId,
+            ...testResultUpdates,
+        }, 5);
     } catch (err) {
         logger.error('Failed to update test finished', { err });
         throw err;
     }
 };
 
-RunStatus.prototype.testEndAndReport = function (wid, result, executionId, sessionId, isRerun) {
+RunStatus.prototype.testEndAndReport = function (wid, result, executionId, sessionId, isRerun, testResultUpdates) {
     const test = this.testEnd(wid, result, executionId, sessionId, isRerun);
-    return this.testEndReport(test, executionId, result);
+    return this.testEndReport(test, executionId, result, testResultUpdates);
 };
 
 RunStatus.prototype.calcTestRunStatus = function () {
@@ -402,10 +396,7 @@ RunStatus.prototype.executionStart = function (executionId, projectId, startTime
                     const ret = servicesApi.reportExecutionStarted(data);
                     this.executionStartedPromise = ret;
                     ret.catch(e => logger.error(e));
-                    if (!this.asyncReporting) {
-                        return ret;
-                    }
-                    return undefined;
+                    return ret;
                 });
             };
 

package/utils.js

@@ -19,7 +19,7 @@ const OSS = [
     { osName: 'Windows 7', bs: { os: 'WINDOWS', os_version: '7' }, sl: { platform: 'Windows 7' } },
     { osName: 'Windows XP', bs: { os: 'WINDOWS', os_version: 'XP' }, sl: { platform: 'Windows XP' } },
     { osName: 'macOS Big Sur', bs: { os: 'OS X', os_version: 'Big Sur', safari_version: '14' }, sl: { platform: 'macOS 11', safari_version: '14' } },
-    { osName: 'macOS Catalina',bs: { os: 'OS X', os_version: 'Catalina', safari_version: '13' }, sl: { platform: 'macOS 10.15', safari_version: '13' } },
+    { osName: 'macOS Catalina', bs: { os: 'OS X', os_version: 'Catalina', safari_version: '13' }, sl: { platform: 'macOS 10.15', safari_version: '13' } },
     { osName: 'macOS Mojave', bs: { os: 'OS X', os_version: 'Mojave', safari_version: '12' }, sl: { platform: 'macOS 10.14', safari_version: '12' } },
     { osName: 'macOS High Sierra', bs: { os: 'OS X', os_version: 'High Sierra', safari_version: '11' }, sl: { platform: 'macOS 10.13', safari_version: '11' } },
     { osName: 'macOS Sierra', bs: { os: 'OS X', os_version: 'Sierra', safari_version: '10' }, sl: { platform: 'macOS 10.12', safari_version: '10.0' } },
@@ -35,11 +35,11 @@ const OSS = [
 ];
 
 const BROWSERS = [
-    { browserName: 'Chrome', bs: { browser: 'Chrome', browser_version: '92' }, sl: { browserName: 'chrome', version: '92.0' }, browserValue: 'chrome' },
+    { browserName: 'Chrome', bs: { browser: 'Chrome', browser_version: '94' }, sl: { browserName: 'chrome', version: '94.0' }, browserValue: 'chrome' },
     { browserName: 'Firefox', bs: { browser: 'Firefox', browser_version: '89' }, sl: { browserName: 'firefox', version: '89.0' }, browserValue: 'firefox' },
     { browserName: 'Safari', bs: { browser: 'Safari' }, sl: { browserName: 'safari' }, browserValue: 'safari' },
-    { browserName: 'Edge', bs: { browser: 'Edge', browser_version: '17' }, sl: { browserName: 'MicrosoftEdge', version: '17.17134' }, browserValue: 'edge' },
-    { browserName: 'Edge Chromium', bs: { browser: 'Edge', browser_version: '92' }, sl: { browserName: 'MicrosoftEdge', version: '92' }, synonyms: ['edge-chromium'], browserValue: 'edge-chromium', seleniumName: 'MicrosoftEdge' },
+    { browserName: 'Edge', bs: { browser: 'Edge', browser_version: '18' }, sl: { browserName: 'MicrosoftEdge', version: '18.17763' }, browserValue: 'edge' },
+    { browserName: 'Edge Chromium', bs: { browser: 'Edge', browser_version: '94' }, sl: { browserName: 'MicrosoftEdge', version: '94' }, synonyms: ['edge-chromium'], browserValue: 'edge-chromium', seleniumName: 'MicrosoftEdge' },
     { browserName: 'Internet Explorer 11', bs: { browser: 'IE', browser_version: '11' }, sl: { browserName: 'internet explorer', version: '11.0' }, synonyms: ['ie11'], browserValue: 'ie11' },
     { browserName: 'Browser', bs: {}, sl: { browserName: 'Browser' }, browserValue: 'browser' },
     { browserName: 'Android', bs: { browserName: 'android' }, sl: {}, browserValue: 'android' },
@@ -143,7 +143,7 @@ function getEnvironmentGitBranch() {
 }
 
 function getUniqBrowsers(options, testList) {
-    if ((options.testConfigNames.length || options.testConfigIds.length || options.testPlan.length || options.testPlanIds.length || options.testObject) && !options.browser) {
+    if ((options.testConfigNames.length || options.testConfigIds.length || options.testPlan.length || options.testPlanIds.length) && !options.browser) {
         return _.uniq(testList.map(t => t.runConfig.browserValue));
     }
     return [options.browser.toLowerCase()];

package/workers/BaseWorker.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const Promise = require('bluebird');
+const Bluebird = require('bluebird');
 const moment = require('moment');
 const ms = require('ms');
 
@@ -38,12 +38,26 @@ function buildFailureResult(testId, testName, resultId, reason) {
 }
 
 class BaseWorker {
-    constructor(executionQueue, customExtensionLocalLocation) {
+    constructor(executionQueue, options, customExtensionLocalLocation, executionId, onTestStarted, onTestCompleted, onGridSlot, onTestIgnored, releaseSlotOnTestFinished = true) {
         this.lambdatestService = new LambdatestService();
+
         this.id = BaseWorker.getWorkerId();
         this.executionQueue = executionQueue;
         this.customExtensionLocalLocation = customExtensionLocalLocation;
-        this.runTest = Promise.method(this.runTest);
+
+        this.isCodeMode = options.files && options.files.length > 0;
+        this.baseUrl = options.baseUrl;
+        this.isRegressionBaselineRun = options.isRegressionBaselineRun;
+        this.testRunTimeout = options.timeout;
+        this.onTestStarted = onTestStarted;
+        this.onTestCompleted = onTestCompleted;
+        this.onGridSlot = onGridSlot;
+        this.onTestIgnored = onTestIgnored;
+        this.releaseSlotOnTestFinished = releaseSlotOnTestFinished;
+
+        this.userData = options.userData;
+        this.executionId = executionId;
+        this.options = options;
     }
 
     static getWorkerId() {
@@ -69,7 +83,7 @@ class BaseWorker {
         throw new NotImplementedError(true);
     }
 
-    runTestOnce(testRunHandler, player) {
+    async runTestOnce(testRunHandler, player) {
         testRunHandler.setSessionId(player.getSessionId());
         logger.info('Test run started', {
             testId: testRunHandler.getTestId(),
@@ -77,26 +91,9 @@ class BaseWorker {
             seleniumSession: player.getSessionId(),
         });
         if (this.options.lightweightMode && this.options.lightweightMode.disableResults) {
-            return Promise.resolve();
+            return undefined;
         }
-        return testRunHandler.clearTestResult();
-    }
-
-    start(options, onStart, onResult, executionId, onGridSlot, onTestIgnored, releaseSlotOnTestFinished) {
-        this.isCodeMode = options.files.length > 0;
-        this.baseUrl = options.baseUrl;
-        this.isRegressionBaselineRun = options.isRegressionBaselineRun;
-        this.testRunTimeout = options.timeout;
-        this.onStart = onStart;
-        this.onResult = onResult;
-        this.onGridSlot = onGridSlot;
-        this.onTestIgnored = onTestIgnored;
-        this.releaseSlotOnTestFinished = releaseSlotOnTestFinished;
-
-        this.userData = options.userData;
-        this.executionId = executionId;
-        this.options = options;
-        return this.run();
+        return await testRunHandler.clearTestResult();
     }
 
     handleQuarantine(testRunHandler) {
@@ -124,13 +121,12 @@ class BaseWorker {
             let gridInfo = await utils.runWithRetries(async () => {
                 const startTime = Date.now();
                 try {
-                    return await Promise.resolve()
-                        .then(() => this.getSlotOnce(testRunHandler))
+                    return await Bluebird.resolve(this.getSlotOnce(testRunHandler))
                         .timeout(this.options.getBrowserTimeout, timeoutMessages.GET_BROWSER_TIMEOUT_MSG);
                 } catch (error) {
                     logger.error('error getting grid slot', { error, testId: this.testId, testResultId: this.testResultId, executionId: this.executionId });
                     failedGetSlotAttempts++;
-                    await Promise.delay(this.options.getBrowserTimeout - (Date.now() - startTime));
+                    await Bluebird.delay(this.options.getBrowserTimeout - (Date.now() - startTime));
                     throw error;
                 }
             }, this.options.getBrowserRetries);
@@ -154,7 +150,7 @@ class BaseWorker {
                     this.options.gridData.host = gridInfo.host;
                     this.options.gridData.failedGetBrowserAttempts = failedGetBrowserAttempts;
                     const getSessionTimeout = Math.max(this.lambdatestService.getSessionTimeout, this.options.getSessionTimeout);
-                    const getBrowserRes = await Promise.resolve()
+                    const getBrowserRes = await Bluebird.resolve()
                         .log('before getBrowserOnce')
                         .then(() => this.getBrowserOnce(testRunHandler, customExtensionLocalLocation, player, gridInfo))
                         .log('after getBrowserOnce')
@@ -169,7 +165,7 @@ class BaseWorker {
                     player.onDone();
 
                     if (!(error instanceof PageNotAvailableError)) {
-                        await Promise.delay(this.options.getBrowserTimeout - (Date.now() - startTime));
+                        await Bluebird.delay(this.options.getBrowserTimeout - (Date.now() - startTime));
                     }
                     throw error;
                 }
@@ -190,14 +186,15 @@ class BaseWorker {
     }
 
     async runTest(testRunHandler, customExtensionLocalLocation, shouldRerun) {
+        perf.log('inside runTest');
         const projectId = this.userData && this.userData.projectId;
         const quarantineResult = this.handleQuarantine(testRunHandler);
         if (quarantineResult) {
             return quarantineResult;
         }
 
-        perf.log('before runTest onStart');
-        await this.onStart(this.id, testRunHandler.getTestId(), testRunHandler.getTestResultId(), shouldRerun, testRunHandler.getRetryKey());
+        perf.log('before runTest onTestStarted');
+        await this.onTestStarted(this.id, testRunHandler.getTestId(), testRunHandler.getTestResultId(), shouldRerun, testRunHandler.getRetryKey());
         const testPlayer = await this.getTestPlayer(testRunHandler, customExtensionLocalLocation);
 
         try {
@@ -207,12 +204,12 @@ class BaseWorker {
         }
     }
 
-    runTestCleanup() {
-        return Promise.resolve();
+    async runTestCleanup() {
+        return undefined;
     }
 
     onQueueCompleted() {
-        return Promise.resolve();
+        return undefined;
     }
 
     run() {
@@ -238,9 +235,9 @@ class BaseWorker {
             try {
                 const testRetryKey = testRunHandler.getRetryKey();
                 testResult.testRetryKey = testRetryKey;
-                await this.onResult(this.id, this.testId, testResult, sessionId, shouldRerun);
-                if (!(this.options.lightweightMode && this.options.lightweightMode.general)) {
-                    await Promise.delay(DELAY_BETWEEN_TESTS);
+                await this.onTestCompleted(this.id, this.testId, testResult, sessionId, shouldRerun);
+                if (this.executionQueue.hasMoreTests() && !(this.options.lightweightMode && this.options.lightweightMode.general)) {
+                    await Bluebird.delay(DELAY_BETWEEN_TESTS);
                 }
                 await this.runTestCleanup();
                 if (shouldRerun) {
@@ -340,7 +337,7 @@ class BaseWorker {
 
             const projectId = this.userData && this.userData.projectId;
             const { errorType, reason } = buildError(err);
-            testResultService.updateTestResult(projectId, this.testResultId, this.testId, {
+            testimServicesApi.updateTestResult(projectId, this.testResultId, this.testId, {
                 status: testRunStatus.COMPLETED,
                 success: false,
                 reason,
@@ -390,10 +387,13 @@ class BaseWorker {
             !disableRemoteStep && remoteStepService.joinToRemoteStep(this.testResultId),
             !disableResults && testResultService.joinToTestResult(this.testResultId, this.testId),
         ])
-            .log('after join room, before runTest')
+            .then(() => testRunHandler.validateRunConfig())
             .then(() => this.runTest(testRunHandler, this.customExtensionLocalLocation, shouldRerun))
             .then(testResult => onRunComplete(testResult, testRunHandler))
-            .log('After onRunComplete')
+            .then(result => {
+                perf.log('After onRunComplete');
+                return result;
+            })
             .catch(runError => recoverTestResults(runError, testRunHandler))
             .finally(() => {
                 if (!disableRemoteStep) {

package/workers/BaseWorker.test.js

@@ -15,7 +15,7 @@ describe('BaseWorker', () => {
     let testPlayerMock;
 
     beforeEach(() => {
-        worker = new BaseWorker();
+        worker = new BaseWorker(null, {});
         worker.userData = {};
         worker.options = { gridData: {}, browser: 'chrome', company: { companyId: 'companyId' }, getBrowserTimeout: 1000, getSessionTimeout: 100, getBrowserRetries: 10 };
         worker.testRunConfig = {};

package/workers/WorkerExtensionSingleBrowser.js

@@ -44,9 +44,8 @@ class WorkerExtensionSingleBrowser extends WorkerExtension {
             return quarantineResult;
         }
 
-        perf.log('before runTest onStart single browser');
-
-        await this.onStart(this.id, testRunHandler.getTestId(), testRunHandler.getTestResultId(), shouldRerun, testRunHandler.getRetryKey());
+        perf.log('before runTest onTestStarted single browser');
+        await this.onTestStarted(this.id, testRunHandler.getTestId(), testRunHandler.getTestResultId(), shouldRerun, testRunHandler.getRetryKey());
         const testPlayer = await this.getTestPlayer(testRunHandler, customExtensionLocalLocation);
 
         testRunHandler.markClearBrowser();
@@ -54,6 +53,10 @@ class WorkerExtensionSingleBrowser extends WorkerExtension {
     }
 
     async runTestCleanup() {
+        if (!this.executionQueue.hasMoreTests()) {
+            await this.onQueueCompleted();
+            return;
+        }
         if (this.options.lightweightMode && this.options.lightweightMode.general) {
             await Bluebird.delay(DELAY_BETWEEN_TESTS);
         }