@testim/testim-cli 3.261.0 → 3.263.0

package/testRunStatus.js

@@ -1,41 +1,50 @@
 'use strict';
 
-const constants = require('./commons/constants');
-const { TESTIM_CONCURRENT_WORKER_COUNT } = require('./commons/config');
+const _ = require('lodash');
 const utils = require('./utils');
-const reporter = require('./reports/reporter.js');
-const servicesApi = require('./commons/testimServicesApi.js');
+const reporter = require('./reports/reporter');
+const constants = require('./commons/constants');
 const gridService = require('./services/gridService');
-const logger = require('./commons/logger').getLogger('test-run-status');
-const { ArgError } = require('./errors');
+const featureFlags = require('./commons/featureFlags');
+const servicesApi = require('./commons/testimServicesApi');
 const OverrideTestDataBuilder = require('./OverrideTestDataBuilder');
-const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
-const Promise = require('bluebird');
-const _ = require('lodash');
+const featureAvailabilityService = require('./commons/featureAvailabilityService');
+const { ArgError } = require('./errors');
+const { getLogger } = require('./commons/logger');
 const { registerExitHook } = require('./processHandler');
 const { calculateCoverage } = require('./coverage/jsCoverage');
-const featureAvailabilityService = require('./commons/featureAvailabilityService');
-const featureFlags = require('./commons/featureFlags');
+const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
 const { mapFilesToLocalDrive } = require('./services/localRCASaver');
-const { promiseMap } = require('./utils');
+const { TESTIM_CONCURRENT_WORKER_COUNT } = require('./commons/config');
 
+const logger = getLogger('test-run-status');
 const gitBranch = utils.getEnvironmentGitBranch();
 const gitCommit = process.env.GIT_COMMIT || process.env.CIRCLE_SHA1 || process.env.TRAVIS_COMMIT;
 const gitRepoUrl = process.env.GIT_URL || process.env.CIRCLE_REPOSITORY_URL;
 const runnerVersion = utils.getRunnerVersion();
 
 
-function runHook(fn, ...args) {
+async function runHook(fn, ...args) {
     if (!fn || typeof fn !== 'function') {
-        return Promise.resolve();
+        return undefined;
     }
-    return Promise.try(() => fn(...args) || {}).catch(err => {
+
+    try {
+        const res = await fn(...args);
+        return res || {};
+    } catch (err) {
         logger.warn('failed to run hook', { err });
         throw new ArgError(`failed to run hook promise ${err.message}`);
-    });
+    }
 }
 
 class RunStatus {
+    /**
+     * @param {any[]} testInfoList
+     * @param {import('./runOptions').RunnerOptions} options
+     * @param {string} testPlanId
+     * @param {string} branchToUse
+     */
     constructor(testInfoList, options, testPlanId, branchToUse) {
         this.options = options;
         this.options.runParams = this.options.runParams || {};
@@ -65,7 +74,7 @@ class RunStatus {
             testPlanId,
             testPlans: options.testPlan,
             testLabels: options.label,
-            testSuites: _.uniq(testInfoList.flatMap(test => test.testSuites)),
+            testSuites: [...new Set(testInfoList.flatMap(test => test.testSuites))],
             testNames: options.name,
             testIds: options.testId,
             testConfigs: options.testConfigNames,
@@ -104,7 +113,18 @@ class RunStatus {
     }) {
         const orgTestResult = this.testRunStatus[originalTestResultId] || {};
         const {
-            config, isTestsContainer, testId, name, testStatus,
+            config,
+            isTestsContainer,
+            testId,
+            name,
+            testStatus,
+            testCreatorName,
+            testCreatorEmail,
+            testOwnerName,
+            testOwnerEmail,
+            testLabels,
+            testSuites,
+            allLabels,
         } = orgTestResult;
 
         const newTestResult = {
@@ -120,7 +140,15 @@ class RunStatus {
             testStatus,
         };
 
-        this.testRunStatus[newResultId] = newTestResult;
+        this.testRunStatus[newResultId] = Object.assign({}, newTestResult, {
+            testCreatorName,
+            testCreatorEmail,
+            testOwnerName,
+            testOwnerEmail,
+            testLabels,
+            testSuites,
+            allLabels,
+        });
 
         return servicesApi.addTestRetry({
             projectId,
@@ -146,55 +174,58 @@ class RunStatus {
         return test;
     }
 
-    updateTestStatusRunning(test, executionId, testRetryKey) {
+    async updateTestStatusRunning(test, executionId, testRetryKey) {
         const { project: projectId, remoteRunId, projectData } = this.options;
         if (this.options.lightweightMode?.onlyTestIdsNoSuite) {
             return this.executionStartedPromise;
         }
 
-        return servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults)
-            .catch(err => {
-                logger.error('failed to upload test data artifact (runner)', { err });
-                return '';
-            })
-            .then(async (testDataUrl) => {
-                const testConfig = _.cloneDeep(test.config);
-                delete testConfig.testData;
-                testConfig.testDataUrl = testDataUrl;
-                await this.executionStartedPromise;
-                return servicesApi.updateTestStatus(projectId, executionId, test.testId, test.resultId, 'RUNNING', { startTime: test.startTime, config: testConfig, remoteRunId, testRetryKey });
-            });
+        let testDataUrl = '';
+
+        try {
+            testDataUrl = await servicesApi.updateTestDataArtifact(projectId, test.testId, test.resultId, test.config.testData, projectData.defaults);
+        } catch (err) {
+            logger.error('failed to upload test data artifact (runner)', { err });
+        }
+        const testConfig = _.cloneDeep(test.config);
+        delete testConfig.testData;
+        testConfig.testDataUrl = testDataUrl;
+        await this.executionStartedPromise;
+        return servicesApi.updateTestStatus(projectId, executionId, test.testId, test.resultId, 'RUNNING', { startTime: test.startTime, config: testConfig, remoteRunId, testRetryKey });
     }
 
-    testStartReport(test, executionId, testRetryKey) {
+    async testStartReport(test, executionId, testRetryKey) {
         if (utils.isQuarantineAndNotRemoteRun(test, this.options)) {
-            return Promise.resolve();
+            return undefined;
+        }
+        const globalParameters = this.exportsGlobal;
+        try {
+            const params = await runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: globalParameters, globalParameters }), this.options.userData.loginData.token);
+
+            // Temporary Sapiens log (SUP-3192)
+            if (this.options.projectData && this.options.projectData.projectId === 'fZ63D61PRQQVvvtGY6Ue' && this.options.suites && this.options.suites.includes('Sanity')) {
+                logger.info('testRunStatus - testStartReport', {
+                    'test.config.testData': test.config.testData,
+                    'this.exportsGlobal': this.exportsGlobal,
+                    'this.fileUserParamsData': this.fileUserParamsData,
+                    'this.beforeSuiteParams': this.beforeSuiteParams,
+                    params,
+                    executionId,
+                    'test.testId': test.testId,
+                    'test.resultId': test.resultId,
+                });
+            }
+
+            test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
+            this.options.runParams[test.resultId] = test.config.testData;
+            test.startTime = Date.now();
+            await this.updateTestStatusRunning(test, executionId, testRetryKey);
+
+            return test;
+        } catch (err) {
+            logger.error('Failed to start test', { err });
+            throw err;
         }
-        return runHook(this.options.beforeTest, Object.assign({}, test, { exportsGlobal: this.exportsGlobal }), this.options.userData.loginData.token)
-            .then(async (params) => {
-                // Temporary Sapiens log (SUP-3192)
-                if (this.options.projectData && this.options.projectData.projectId === 'fZ63D61PRQQVvvtGY6Ue' && this.options.suites && this.options.suites.includes('Sanity')) {
-                    logger.info('testRunStatus - testStartReport', {
-                        'test.config.testData': test.config.testData,
-                        'this.exportsGlobal': this.exportsGlobal,
-                        'this.fileUserParamsData': this.fileUserParamsData,
-                        'this.beforeSuiteParams': this.beforeSuiteParams,
-                        params,
-                        executionId,
-                        'test.testId': test.testId,
-                        'test.resultId': test.resultId,
-                    });
-                }
-                test.config.testData = Object.assign({}, test.config.testData, this.exportsGlobal, this.fileUserParamsData, this.beforeSuiteParams, params);
-                this.options.runParams[test.resultId] = test.config.testData;
-                test.startTime = Date.now();
-                await this.updateTestStatusRunning(test, executionId, testRetryKey);
-
-                return test;
-            }).catch(err => {
-                logger.error('Failed to start test', { err });
-                throw err;
-            });
     }
 
     testStartAndReport(wid, executionId, resultId, isRerun, testRetryKey) {
@@ -219,16 +250,14 @@ class RunStatus {
             reporter.onTestPassed(name);
             return;
         }
-        reporter.onTestFailed(test,
+        reporter.onTestFailed(
+            test,
             test.failureReason,
-            utils.getTestUrl(this.options.editorUrl,
-                this.options.project,
-                testId,
-                resultId,
-                this.branchToUse),
+            utils.getTestUrl(this.options.editorUrl, this.options.project, testId, resultId, this.branchToUse),
             testId,
             isRerun,
-            resultId);
+            resultId,
+        );
     }
 
     calcResultText(result) {
@@ -298,7 +327,7 @@ class RunStatus {
         const globalParameters = result.exportsGlobal;
         try {
             try {
-                await runHook(this.options.afterTest, Object.assign({}, test, { globalParameters }), this.options.userData.loginData.token);
+                await runHook(this.options.afterTest, Object.assign({}, test, { exportsGlobal: globalParameters, globalParameters }), this.options.userData.loginData.token);
             } catch (err) {
                 logger.error('HOOK threw an error', { test: test.testId, err });
                 // eslint-disable-next-line no-console
@@ -349,6 +378,10 @@ class RunStatus {
 
             const runConfig = options.browser ? utils.getRunConfigByBrowserName(options.browser, options.saucelabs, options.browserstack) : testInfo.runConfig;
 
+            if (runConfig && featureFlags.flags.dec2022eolBrowsers.isEnabled() && utils.getBrowserInfo(runConfig.browserValue)?.eol) {
+                throw new ArgError(`Unsupported Browser: ${runConfig.browserName}`);
+            }
+
             resultStatus[testInfo.resultId].config = Object.assign({}, this.execConfig, {
                 companyId,
                 testData: testInfo.testData?.value ? testInfo.testData.value : null,
@@ -364,7 +397,7 @@ class RunStatus {
         }, {});
     }
 
-    executionStart(executionId, projectId, startTime, testPlanName, testNames) {
+    async executionStart(executionId, projectId, startTime, testPlanName, testNames) {
         logger.info('execution started', { executionId });
         const { options } = this;
         const { remoteRunId, projectData } = options;
@@ -383,66 +416,74 @@ class RunStatus {
         ]));
 
         this.startTime = startTime || Date.now();
-        const runHooksProps = { projectId, executionId };
-        if (featureFlags.flags.testNamesToBeforeSuiteHook.isEnabled()) {
-            runHooksProps.testNames = testNames;
+        const runHooksProps = {
+            projectId,
+            executionId,
+            ...(featureFlags.flags.testNamesToBeforeSuiteHook.isEnabled() && { testNames }),
+        };
+        const params = await runHook(options.beforeSuite, runHooksProps);
+        const overrideTestDataBuilder = new OverrideTestDataBuilder(params, _.cloneDeep(this.testInfoList), projectId);
+        this.testInfoList = overrideTestDataBuilder.overrideTestData();
+        this.calcTestRunStatus();
+        this.beforeSuiteParams = params;
+
+        const { testInfoList } = this;
+        const beforeTests = [];
+        const tests = [];
+        const afterTests = [];
+        for (const test of testInfoList) {
+            if (test.isBeforeTestPlan) {
+                beforeTests.push(test);
+                continue;
+            }
+            if (test.isAfterTestPlan) {
+                afterTests.push(test);
+                continue;
+            }
+            tests.push(test);
         }
-        return runHook(options.beforeSuite, runHooksProps)
-            .then(params => {
-                const overrideTestDataBuilder = new OverrideTestDataBuilder(params, _.cloneDeep(this.testInfoList), projectId);
-                this.testInfoList = overrideTestDataBuilder.overrideTestData();
-                this.calcTestRunStatus();
-                this.beforeSuiteParams = params;
-
-                const { testInfoList } = this;
-                const beforeTests = testInfoList.filter(test => test.isBeforeTestPlan);
-                const tests = testInfoList.filter(test => !test.isBeforeTestPlan && !test.isAfterTestPlan);
-                const afterTests = testInfoList.filter(test => test.isAfterTestPlan);
-
-                const reportExecutionStarted = () => {
-                    const testResults = _.cloneDeep(this.testRunStatus);
-                    return promiseMap(Object.keys(testResults), testResultId => {
-                        const test = testResults[testResultId];
-                        const testData = test.config?.testData;
-                        const testId = test.testId;
-                        return servicesApi.updateTestDataArtifact(projectId, testId, testResultId, testData, projectData.defaults)
-                            .then((testDataUrl) => {
-                                if (!testDataUrl) {
-                                    return;
-                                }
-                                delete test.config.testData;
-                                test.config.testDataUrl = testDataUrl;
-                            });
-                    }).then(() => {
-                        const isLocalRun = Boolean(options.useLocalChromeDriver || options.useChromeLauncher);
-                        const data = {
-                            executionId,
-                            projectId,
-                            labels: testPlanName || [],
-                            startTime,
-                            executions: testResults,
-                            config: this.execConfig,
-                            resultLabels: options.resultLabels,
-                            remoteRunId: options.remoteRunId,
-                            localRunUserId: options.user,
-                            isLocalRun,
-                            intersections: options.intersections,
-                        };
-                        const ret = servicesApi.reportExecutionStarted(data);
-                        this.executionStartedPromise = ret;
-                        ret.catch(e => logger.error(e));
-                        return ret;
-                    });
-                };
-
-                return reportExecutionStarted()
-                    .catch(err => {
-                        logger.error('Failed to start suite', { err });
-                        // eslint-disable-next-line no-console
-                        console.error('Failed to start test run. Please contact support@testim.io');
-                    })
-                    .then(() => ({ beforeTests, tests, afterTests }));
+
+        const reportExecutionStarted = async () => {
+            const testResults = _.cloneDeep(this.testRunStatus);
+            await utils.promiseMap(Object.keys(testResults), async testResultId => {
+                const test = testResults[testResultId];
+                const testData = test.config?.testData;
+                const testId = test.testId;
+                const testDataUrl = await servicesApi.updateTestDataArtifact(projectId, testId, testResultId, testData, projectData.defaults);
+                if (!testDataUrl) {
+                    return;
+                }
+                delete test.config.testData;
+                test.config.testDataUrl = testDataUrl;
             });
+            const isLocalRun = Boolean(options.useLocalChromeDriver || options.useChromeLauncher);
+            const data = {
+                executionId,
+                projectId,
+                labels: testPlanName || [],
+                startTime,
+                executions: testResults,
+                config: this.execConfig,
+                resultLabels: options.resultLabels,
+                remoteRunId: options.remoteRunId,
+                localRunUserId: options.user,
+                isLocalRun,
+                intersections: options.intersections,
+            };
+            const ret = servicesApi.reportExecutionStarted(data);
+            this.executionStartedPromise = ret;
+            ret.catch(e => logger.error(e));
+            return ret;
+        };
+
+        try {
+            await reportExecutionStarted();
+        } catch (err) {
+            logger.error('Failed to start suite', { err });
+            // eslint-disable-next-line no-console
+            console.error('Failed to start test run. Please contact support@testim.io');
+        }
+        return { beforeTests, tests, afterTests };
     }
 
     concatSeleniumPerfMarks(marks) {
@@ -456,47 +497,58 @@ class RunStatus {
             .value();
     }
 
-    executionEnd(executionId) {
+    async executionEnd(executionId) {
         const tests = utils.groupTestsByRetries(this.testRunStatus);
         const total = tests.length;
-        const passed = tests.filter(({ status }) => status === constants.runnerTestStatus.PASSED).length;
-        const skipped = tests.filter(({ status }) => status === constants.runnerTestStatus.SKIPPED).length;
-        const failedInEvaluatingStatus = tests.filter(({ status, testStatus }) => status === constants.runnerTestStatus.FAILED && testStatus === constants.testStatus.EVALUATING).length;
 
-        const resultExtraData = { ...this.seleniumPerfStats.getStats() };
-        delete resultExtraData.seleniumPerfMarks;
+        let passed = 0;
+        let skipped = 0;
+        let failedInEvaluatingStatus = 0;
+        for (const { status, testStatus } of tests) {
+            if (status === constants.runnerTestStatus.PASSED) {
+                passed++;
+            }
+            if (status === constants.runnerTestStatus.SKIPPED) {
+                skipped++;
+            }
+            if (status === constants.runnerTestStatus.FAILED && testStatus === constants.testStatus.EVALUATING) {
+                failedInEvaluatingStatus++;
+            }
+        }
+
+        const { seleniumPerfMarks, ...resultExtraData } = this.seleniumPerfStats.getStats();
 
-        return runHook(this.options.afterSuite, {
+        await runHook(this.options.afterSuite, {
             exportsGlobal: this.exportsGlobal,
             tests,
             total,
             passed,
             skipped,
-        })
-            .then(() => calculateCoverage(this.options, this.branchToUse, total, executionId))
-            .then((coverageSummary) => {
-                resultExtraData.coverageSummary = coverageSummary;
+        });
+        const coverageSummary = await calculateCoverage(this.options, this.branchToUse, total, executionId);
+        resultExtraData.coverageSummary = coverageSummary;
 
-                if (this.options.lightweightMode?.onlyTestIdsNoSuite) {
-                    return undefined;
-                }
-                return servicesApi.reportExecutionFinished(
-                    'FINISHED',
-                    executionId,
-                    this.options.project,
-                    total === (passed + skipped + failedInEvaluatingStatus),
-                    {
-                        tmsSuppressReporting: this.options.tmsSuppressReporting,
-                        tmsRunId: this.options.tmsRunId,
-                        tmsCustomFields: this.options.tmsCustomFields,
-                    },
-                    this.options.remoteRunId,
-                    resultExtraData
-                ).catch(err => {
-                    logger.error('Failed to update suite finished', { err });
-                    throw err;
-                });
-            });
+        if (this.options.lightweightMode?.onlyTestIdsNoSuite) {
+            return undefined;
+        }
+        try {
+            return await servicesApi.reportExecutionFinished(
+                'FINISHED',
+                executionId,
+                this.options.project,
+                total === (passed + skipped + failedInEvaluatingStatus),
+                {
+                    tmsSuppressReporting: this.options.tmsSuppressReporting,
+                    tmsRunId: this.options.tmsRunId,
+                    tmsCustomFields: this.options.tmsCustomFields,
+                },
+                this.options.remoteRunId,
+                resultExtraData
+            );
+        } catch (err) {
+            logger.error('Failed to update suite finished', { err });
+            throw err;
+        }
     }
 
     async markAllQueuedTests(executionId, status, failureReason, success) {

package/utils/index.js

@@ -43,16 +43,22 @@ const BROWSERS = [
     { browserName: 'Chrome', bs: { browser: 'Chrome', browser_version: '94' }, sl: { browserName: 'chrome', version: '94.0' }, browserValue: 'chrome' },
     { browserName: 'Firefox', bs: { browser: 'Firefox', browser_version: '89' }, sl: { browserName: 'firefox', version: '89.0' }, browserValue: 'firefox' },
     { browserName: 'Safari', bs: { browser: 'Safari' }, sl: { browserName: 'safari' }, browserValue: 'safari' },
-    { browserName: 'Edge', bs: { browser: 'Edge', browser_version: '18' }, sl: { browserName: 'MicrosoftEdge', version: '18.17763' }, browserValue: 'edge' },
+    { browserName: 'Edge', bs: { browser: 'Edge', browser_version: '18' }, sl: { browserName: 'MicrosoftEdge', version: '18.17763' }, browserValue: 'edge', eol: true },
     // eslint-disable-next-line max-len
     { browserName: 'Edge Chromium', bs: { browser: 'Edge', browser_version: '94' }, sl: { browserName: 'MicrosoftEdge', version: '94' }, synonyms: ['edge-chromium'], browserValue: 'edge-chromium', seleniumName: 'MicrosoftEdge' },
-    { browserName: 'Internet Explorer 11', bs: { browser: 'IE', browser_version: '11' }, sl: { browserName: 'internet explorer', version: '11.0' }, synonyms: ['ie11'], browserValue: 'ie11' },
+    // eslint-disable-next-line max-len
+    { browserName: 'Internet Explorer 11', bs: { browser: 'IE', browser_version: '11' }, sl: { browserName: 'internet explorer', version: '11.0' }, synonyms: ['ie11'], browserValue: 'ie11', eol: true },
     { browserName: 'Browser', bs: {}, sl: { browserName: 'Browser' }, browserValue: 'browser' },
     { browserName: 'Android', bs: { browserName: 'android' }, sl: {}, browserValue: 'android' },
     { browserName: 'iPad', bs: { browserName: 'iPad' }, sl: {}, browserValue: 'ipad' },
     { browserName: 'iPhone', bs: { browserName: 'iPhone' }, sl: {}, browserValue: 'iphone' },
 ];
 
+function getBrowserInfo(browserValue) {
+    browserValue = browserValue.toLowerCase();
+    return BROWSERS.find((b) => b.browserValue === browserValue);
+}
+
 function getRunConfigByBrowserName(browser, saucelabs, browserstack) {
     browser = browser.toLowerCase();
     const selectedBrowser = BROWSERS.find(b => b.browserName.toLowerCase() === browser || browser.includes(b.synonyms)) || BROWSERS[0];
@@ -182,6 +188,7 @@ module.exports = {
     groupTestsByRetries,
     getPlanType,
     getCdpAddressForHost,
+    getBrowserInfo,
     ...fsUtils,
     ...argsUtils,
     ...timeUtils,

package/workers/BaseWorker.js

@@ -38,6 +38,17 @@ function buildFailureResult(testId, testName, resultId, reason) {
 }
 
 class BaseWorker {
+    /**
+     * @param {import('../executionQueue')} executionQueue
+     * @param {import('../runOptions').RunnerOptions} options
+     * @param {string=} customExtensionLocalLocation
+     * @param {string} executionId
+     * @param {Function} onTestStarted
+     * @param {Function} onTestCompleted
+     * @param {Function} onGridSlot
+     * @param {Function} onTestIgnored
+     * @param {boolean=} releaseSlotOnTestFinished
+     */
     constructor(executionQueue, options, customExtensionLocalLocation, executionId, onTestStarted, onTestCompleted, onGridSlot, onTestIgnored, releaseSlotOnTestFinished = true) {
         this.lambdatestService = new LambdatestService();