@testim/testim-cli 3.233.0 → 3.236.0

package/runners/TestPlanRunner.js

@@ -20,7 +20,6 @@ const { getSuite, calcTestResultStatus, validateConfig } = require('./runnerUtil
 const { StopRunOnError, ArgError } = require('../errors');
 const Logger = require('../commons/logger');
 const perf = require('../commons/performance-logger');
-const featureFlags = require('../commons/featureFlags');
 
 const guid = utils.guid;
 const logger = Logger.getLogger('test-plan-runner');
@@ -35,14 +34,14 @@ class TestPlanRunner {
         const executionResults = {};
         const authData = testimCustomToken.getTokenV3UserData();
 
-        const runBeforeTests = (beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
+        const runBeforeTests = () => {
             const workerCount = 1;
             const stopOnError = true;
             return this.workerManager.runTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(beforeTestsResults => Object.assign(executionResults, beforeTestsResults));
         };
 
-        const runTestPlanTests = (tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
+        const runTestPlanTests = () => {
             const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
             const stopOnError = false;
             perf.log('right before this.workerManager.runTests');
@@ -51,30 +50,30 @@ class TestPlanRunner {
                 .then(testsResults => Object.assign(executionResults, testsResults));
         };
 
-        const runAfterTests = (afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
+        const runAfterTests = () => {
             const workerCount = 1;
             const stopOnError = false;
             return this.workerManager.runTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(afterTestsResults => Object.assign(executionResults, afterTestsResults));
         };
 
-        function catchBeforeTestsFailed(executionId) {
+        function catchBeforeTestsFailed() {
             return testStatus.markAllQueuedTests(executionId, constants.runnerTestStatus.ABORTED, 'aborted', false);
         }
 
         const sessionType = utils.getSessionType(tpOptions);
         analyticsService.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
         perf.log('right before runBeforeTests');
-        return runBeforeTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData)
+        return runBeforeTests()
             .log('right before runTestPlanTests')
-            .then(() => runTestPlanTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
+            .then(() => runTestPlanTests())
             .log('right after runTestPlanTests')
-            .then(() => runAfterTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
+            .then(() => runAfterTests())
             .then(() => executionResults)
             .catch(err => {
                 logger.error('error running test plan', { err });
                 if (err instanceof StopRunOnError) {
-                    return catchBeforeTestsFailed(executionId);
+                    return catchBeforeTestsFailed();
                 }
                 throw err;
             })

package/services/lambdatestService.js

@@ -1,13 +1,12 @@
 const os = require('os');
-const { spawn } = require('child_process');
+const childProcess = require('child_process');
 const Promise = require('bluebird');
 const fse = require('fs-extra');
 const portfinder = require('portfinder');
 const ms = require('ms');
 
-const {
-    downloadAndSave, unzipFile, guid, runWithRetries, isURL,
-} = require('../utils');
+const { guid, isURL } = require('../utils');
+const utils = require('../utils');
 const { gridTypes, CLI_MODE } = require('../commons/constants');
 const httpRequest = require('../commons/httpRequest');
 const { ArgError } = require('../errors');
@@ -86,8 +85,8 @@ class LambdatestService {
             throw new Error(`tunnel on ${os.platform() + os.arch()} platform is not supported.`);
         }
         const zipLocation = `${LT_TUNNEL_BINARY_DIRECTORY}.zip`;
-        await downloadAndSave(`${LT_TUNNEL_BINARY_ORIGIN}/${downloadUrl}`, zipLocation);
-        await unzipFile(zipLocation, LT_TUNNEL_BINARY_DIRECTORY);
+        await utils.downloadAndSave(`${LT_TUNNEL_BINARY_ORIGIN}/${downloadUrl}`, zipLocation);
+        await utils.unzipFile(zipLocation, LT_TUNNEL_BINARY_DIRECTORY);
     }
 
     static async connectTunnel(runnerOptions) {
@@ -136,7 +135,11 @@ class LambdatestService {
             }
         }
 
-        LambdatestService.tunnel = spawn('./LT', tunnelArgs, { cwd: LT_TUNNEL_BINARY_DIRECTORY });
+        if (runnerOptions.externalLambdatestMitm) {
+            tunnelArgs = [...tunnelArgs, '--mitm'];
+        }
+
+        LambdatestService.tunnel = childProcess.spawn('./LT', tunnelArgs, { cwd: LT_TUNNEL_BINARY_DIRECTORY });
 
         let stdoutResult = '';
         let stderrResult = '';
@@ -151,7 +154,7 @@ class LambdatestService {
 
         // verify that LT tunnel strated successfully
         try {
-            const ltInfo = await runWithRetries(() => httpRequest.get(`http://127.0.0.1:${infoAPIPort}/api/v1.0/info`, {}, {}, undefined, { skipProxy: true }), 30, 2000);
+            const ltInfo = await utils.runWithRetries(() => httpRequest.get(`http://127.0.0.1:${infoAPIPort}/api/v1.0/info`, {}, {}, undefined, { skipProxy: true }), 30, 2000);
             logger.info('LT tunnel info', ltInfo);
         } catch (err) {
             logger.error('Failed to start LT tunnel', { err, stdoutResult, stderrResult });
@@ -166,7 +169,7 @@ class LambdatestService {
         return new Promise((resolve, reject) => {
             LambdatestService.tunnel.on('close', (code) => {
                 if (code) {
-                    reject();
+                    reject(new Error(`tunnel process exited with code ${code}`));
                 }
                 resolve();
             });
@@ -207,6 +210,7 @@ class LambdatestService {
             build: executionId,
             name: `${testResultId} - ${testName}`,
             platform: LambdatestService.lambdatestConfig.PLATFORM,
+            // eslint-disable-next-line camelcase
             selenium_version: LambdatestService.lambdatestConfig.SELENIUM_VERSION,
             resolution: LambdatestService.lambdatestConfig.RESOLUTION,
             timezone: LambdatestService.lambdatestConfig.TIMEZONE,

package/services/lambdatestService.test.js

@@ -1,11 +1,38 @@
 const { expect, sinon } = require('../../test/utils/testUtils');
 const servicesApi = require('../commons/testimServicesApi');
+const httpRequest = require('../commons/httpRequest');
 const LambdatestService = require('./lambdatestService');
+const utils = require('../utils');
+const fse = require('fs-extra');
+const os = require('os');
+const childProcess = require('child_process');
+const EventEmitter = require('events');
+const portfinder = require('portfinder');
+const { getExtensionsUrl } = require('../runOptionsUtils');
+
+class Process extends EventEmitter {
+    constructor() {
+        super();
+        this.stdout = new EventEmitter();
+        this.stderr = new EventEmitter();
+    }
+    setCode(code) {
+        this.code = code;
+    }
+    kill() {
+        this.emit('close', this.code);
+    }
+}
 
 describe('LambdatestService', () => {
+    let sandbox;
     let lambdatestService;
     beforeEach(() => {
         lambdatestService = new LambdatestService();
+        sandbox = sinon.createSandbox();
+    });
+    afterEach(() => {
+        sandbox.restore();
     });
 
     describe('isLambdatestGrid', () => {
@@ -86,8 +113,240 @@ describe('LambdatestService', () => {
             lambdatestService.isActive = false;
             expect(lambdatestService.getSessionRetries).to.be.equal(null);
         });
+
+        it('should not return lt special selenium capabilities when inactove', () => {
+            lambdatestService.isActive = false;
+            expect(lambdatestService.getCapabilities({})).to.eql({});
+        });
+
+        it('should return lt selenium capabilities', () => {
+            const browser = utils.guid();
+            const executionId = utils.guid();
+            const testResultId = utils.guid();
+            const testName = utils.guid();
+
+            lambdatestService.isActive = true;
+            LambdatestService.lambdatestConfig = {
+                CAPABILITIES: { [browser]: { specificBrowserCaps: 123 } },
+            };
+
+            expect(lambdatestService.getCapabilities({}, browser, executionId, testResultId, testName)).to.shallowDeepEqual({
+                build: executionId,
+                name: `${testResultId} - ${testName}`,
+                platform: LambdatestService.lambdatestConfig.PLATFORM,
+                // eslint-disable-next-line camelcase
+                selenium_version: LambdatestService.lambdatestConfig.SELENIUM_VERSION,
+                resolution: LambdatestService.lambdatestConfig.RESOLUTION,
+                timezone: LambdatestService.lambdatestConfig.TIMEZONE,
+                specificBrowserCaps: 123,
+                console: true,
+                queueTimeout: 300,
+            });
+        });
+
+        it('should return lt tunnel name as part of selenium capabilities', () => {
+            lambdatestService.isActive = true;
+            LambdatestService.lambdatestConfig = { CAPABILITIES: {} };
+            LambdatestService.tunnelName = utils.guid();
+
+            expect(lambdatestService.getCapabilities({})).to.shallowDeepEqual({
+                tunnel: true,
+                tunnelName: LambdatestService.tunnelName,
+            });
+        });
+
+        it('should load testim extension when it is not set', () => {
+            lambdatestService.isActive = true;
+            LambdatestService.lambdatestConfig = { CAPABILITIES: {} };
+            LambdatestService.tunnelName = utils.guid();
+
+            expect(lambdatestService.getCapabilities({ mode: 'extension' }, 'chrome')).to.shallowDeepEqual({ loadExtension: [getExtensionsUrl({}, true).chrome] });
+            expect(lambdatestService.getCapabilities({ mode: 'extension' }, 'somOtherBrowser')).to.shallowDeepEqual({ loadExtension: [] });
+        });
+
+        it('should load testim extension when passing extensionPath flag', () => {
+            lambdatestService.isActive = true;
+            LambdatestService.lambdatestConfig = { CAPABILITIES: {} };
+            LambdatestService.tunnelName = utils.guid();
+
+            const extensionPath = 'http://localhost:1234/extension.zip';
+            expect(lambdatestService.getCapabilities({ mode: 'extension', extensionPath })).to.shallowDeepEqual({ loadExtension: [extensionPath] });
+        });
+
+        it('should load testim extension when passing installCustomExtension flag', () => {
+            lambdatestService.isActive = true;
+            LambdatestService.lambdatestConfig = { CAPABILITIES: {} };
+            LambdatestService.tunnelName = utils.guid();
+
+            const installCustomExtension = 'http://localhost:1234/extension.zip';
+            expect(lambdatestService.getCapabilities({ mode: 'extension', installCustomExtension })).to.shallowDeepEqual({ loadExtension: [installCustomExtension] });
+        });
+    });
+
+    describe('prepareTunnel', () => {
+        it('should do nothing when tunnel binary already exists', async () => {
+            sandbox.stub(fse, 'pathExists').resolves(true);
+            const chmod = sandbox.stub(fse, 'chmodSync');
+            await LambdatestService.prepareTunnel();
+            expect(chmod).not.to.have.been.called;
+        });
+
+        it('should throw when unsupported os', async () => {
+            sandbox.stub(fse, 'pathExists').resolves(false);
+            sandbox.stub(os, 'platform').returns('wtf');
+            sandbox.stub(os, 'arch').returns('wtf');
+
+            await expect(LambdatestService.prepareTunnel()).to.be.rejectedWith(Error, 'tunnel on wtfwtf platform is not supported.');
+        });
+
+        it('should download and extract tunnel binary', async () => {
+            sandbox.stub(fse, 'pathExists').resolves(false);
+            sandbox.stub(os, 'platform').returns('win32');
+            sandbox.stub(os, 'arch').returns('x64');
+
+            const download = sandbox.stub(utils, 'downloadAndSave');
+            const unzip = sandbox.stub(utils, 'unzipFile').resolves();
+
+            await LambdatestService.prepareTunnel();
+            sinon.assert.calledOnce(unzip);
+            sinon.assert.calledOnce(download);
+            expect(download.args[0][0]).to.startWith('https://downloads.lambdatest.com/tunnel/');
+        });
     });
 
     describe('connectTunnel', () => {
+        let prepareTunnelStub;
+        let spawnStub;
+        let credentials;
+        let httpGetStub;
+        let processMock;
+
+        beforeEach(() => {
+            processMock = new Process();
+            credentials = { gridUsername: utils.guid(), gridPassword: utils.guid() };
+            sandbox.stub(portfinder, 'getPortPromise').resolves(1234);
+            prepareTunnelStub = sandbox.stub(LambdatestService, 'prepareTunnel');
+            spawnStub = sandbox.stub(childProcess, 'spawn').returns(processMock);
+            httpGetStub = sandbox.stub(httpRequest, 'get').resolves({});
+        });
+
+        it('should do nothing when using externalLambdatestTunnelId', async () => {
+            await LambdatestService.connectTunnel({ externalLambdatestTunnelId: 123 });
+            sinon.assert.neverCalledWith(prepareTunnelStub);
+            sinon.assert.neverCalledWith(spawnStub);
+        });
+
+        it('should prepare the tunnel', async () => {
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(prepareTunnelStub);
+        });
+
+        it('should reject when no credentials', async () => {
+            await expect(LambdatestService.connectTunnel({ })).to.be.rejectedWith(Error, 'tunnel requires username and password');
+        });
+
+        it('should spawn the tunnel process', async () => {
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1]).to.eql([
+                '--tunnelName', LambdatestService.tunnelName, '--infoAPIPort', 1234,
+                '--user', credentials.gridUsername, '--key', credentials.gridPassword,
+            ]);
+        });
+
+        it('should accept tunnelUser and tunnelKey on gridData', async () => {
+            credentials = { gridData: { tunnelUser: utils.guid(), tunnelKey: utils.guid() } };
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1]).to.eql([
+                '--tunnelName', LambdatestService.tunnelName, '--infoAPIPort', 1234,
+                '--user', credentials.gridData.tunnelUser, '--key', credentials.gridData.tunnelKey,
+            ]);
+        });
+
+        it('should allow using externalLambdatestUseWss', async () => {
+            await LambdatestService.connectTunnel({ ...credentials, externalLambdatestUseWss: true });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--mode ws');
+        });
+
+        it('should allow using proxyUri', async () => {
+            global.proxyUri = 'http://localhost';
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--proxy-host localhost');
+            global.proxyUri = undefined;
+        });
+
+        it('should allow using proxyUri port and credentials', async () => {
+            global.proxyUri = 'http://user:pass@localhost:1234';
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--proxy-host localhost');
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--proxy-port 1234');
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--proxy-user user');
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--proxy-pass pass');
+            global.proxyUri = undefined;
+        });
+
+        it('should throw when proxyUri is invalid', async () => {
+            global.proxyUri = 'i am invalid';
+            await expect(LambdatestService.connectTunnel({ ...credentials })).to.be.rejectedWith(Error, 'proxy url is invalid');
+            global.proxyUri = undefined;
+        });
+
+        it('should allow using externalLambdatestDisableAutomationTunneling', async () => {
+            await LambdatestService.connectTunnel({ ...credentials, externalLambdatestDisableAutomationTunneling: true });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--bypassHosts run.testim.io,services.testim.io,api.coralogix.com,conf.rollout.io,statestore.rollout.io,push.rollout.io,analytic.rollout.io,res.cloudinary.com');
+        });
+
+        it('should allow using externalLambdatestMitm', async () => {
+            await LambdatestService.connectTunnel({ ...credentials, externalLambdatestMitm: true });
+            sinon.assert.calledOnce(spawnStub);
+            expect(spawnStub.args[0][1].join(' ')).to.contain('--mitm');
+        });
+
+        it('should verify tunnel started', async () => {
+            await LambdatestService.connectTunnel({ ...credentials });
+            sinon.assert.calledOnce(httpGetStub);
+        });
+        it('should throw when tunnel did not start', async () => {
+            sandbox.stub(utils, 'runWithRetries').rejects(new Error('tunnel did not start'));
+            await expect(LambdatestService.connectTunnel({ ...credentials })).to.be.rejectedWith(Error, 'tunnel did not start');
+            processMock.stdout.emit('data', '');
+            processMock.stderr.emit('data', '');
+        });
+    });
+
+    describe('disconnectTunnel', () => {
+        let processMock;
+        let killStub;
+
+        beforeEach(() => {
+            processMock = new Process();
+            killStub = sandbox.stub(processMock, 'kill').callThrough();
+            sandbox.stub(childProcess, 'spawn').returns(processMock);
+            sandbox.stub(LambdatestService, 'prepareTunnel');
+            sandbox.stub(httpRequest, 'get').resolves({});
+        });
+
+        it('should kill the tunnel', async () => {
+            await LambdatestService.connectTunnel({ tunnel: true, company: {}, gridUsername: utils.guid(), gridPassword: utils.guid() });
+            await LambdatestService.disconnectTunnel({ company: {} });
+            sinon.assert.calledOnce(killStub);
+        });
+
+        it('should reject when killing the tunnel fails', async () => {
+            processMock.setCode(1);
+            await LambdatestService.connectTunnel({ tunnel: true, company: {}, gridUsername: utils.guid(), gridPassword: utils.guid() });
+            await expect(LambdatestService.disconnectTunnel({ company: {} })).to.be.rejectedWith(Error);
+            sinon.assert.calledOnce(killStub);
+        });
+
+        it('should do nothing when using externalLambdatestTunnelId', async () => {
+            await LambdatestService.disconnectTunnel({ externalLambdatestTunnelId: 123 });
+            sinon.assert.neverCalledWith(killStub);
+        });
     });
 });

package/testRunStatus.js

@@ -174,7 +174,21 @@ RunStatus.prototype.testStartReport = function (test, executionId, testRetryKey)
     }
     return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }), this.options.userData.loginData.token)
         .then(async params => {
-            this.options.runParams[test.resultId] = test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
+            // Temporary Sapiens log (SUP-3192)
+            if (this.options.projectData && this.options.projectData.projectId === 'fZ63D61PRQQVvvtGY6Ue' && this.options.suites && this.options.suites.includes('Sanity')) {
+                logger.info('testRunStatus - testStartReport', {
+                    'test.config.testData': test.config.testData,
+                    'this.exportsGlobal': this.exportsGlobal,
+                    'this.fileUserParamsData': this.fileUserParamsData,
+                    'this.beforeSuiteParams': this.beforeSuiteParams,
+                    params,
+                    executionId,
+                    'test.testId': test.testId,
+                    'test.resultId': test.resultId,
+                });
+            }
+            test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
+            this.options.runParams[test.resultId] = test.config.testData;
             test.startTime = Date.now();
             await this.updateTestStatusRunning(test, executionId, testRetryKey);
 
@@ -234,8 +248,10 @@ RunStatus.prototype.testEnd = function (wid, result, executionId, sessionId, isR
     const duration = (result.endTime - result.startTime) || 0;
     test.sessionId = sessionId;
     test.startTime = result.startTime || test.startTime || Date.now();
-    test.duration = result.duration = duration;
-    result.failureReason = test.failureReason = result.failureReason || result.reason;
+    test.duration = duration;
+    result.duration = duration;
+    test.failureReason = result.failureReason || result.reason;
+    result.failureReason = test.failureReason;
     test.failurePath = result.failurePath;
     test.resultId = result.resultId;
     test.success = result.success;
@@ -245,8 +261,9 @@ RunStatus.prototype.testEnd = function (wid, result, executionId, sessionId, isR
     }
 
     test.resultUrl = utils.getTestUrl(this.options.editorUrl, this.options.project, test.testId, test.resultId, this.branchToUse);
+    test.status = this.calcResultText(result);
 
-    test.status = result.status = this.calcResultText(result);
+    result.status = test.status;
     result.name = test.name;
     result.testStatus = test.testStatus;
     result.testId = result.testId || test.testId;
@@ -263,7 +280,19 @@ RunStatus.prototype.testEnd = function (wid, result, executionId, sessionId, isR
     const isCodeMode = this.options.files.length > 0;
     reporter.onTestFinished(test, wid, isRerun, isCodeMode);
 
-    this.exportsGlobal = Object.assign({}, this.exportsGlobal, result.exportsGlobal);
+    const afterMerge = Object.assign({}, this.exportsGlobal, result.exportsGlobal);
+    // Temporary Sapiens log (SUP-3192)
+    if (this.options.projectData && this.options.projectData.projectId === 'fZ63D61PRQQVvvtGY6Ue' && this.options.suites && this.options.suites.includes('Sanity')) {
+        logger.info('testRunStatus - testEnd', {
+            'this.exportsGlobal': this.exportsGlobal,
+            'result.exportsGlobal': result.exportsGlobal,
+            afterMerge,
+            executionId,
+            'test.testId': test.testId,
+            'test.resultId': test.resultId,
+        });
+    }
+    this.exportsGlobal = afterMerge;
     return test;
 };