@testim/testim-cli 3.196.0 → 3.197.0

package/runners/AnonymousTestPlanRunner.js

@@ -1,106 +0,0 @@
-'use strict';
-
-const path = require('path');
-const Promise = require('bluebird');
-
-const _ = require('lodash');
-
-const reporter = require('../reports/reporter');
-const logger = require('../commons/logger').getLogger('suite-runner');
-const utils = require('../utils.js');
-const { ArgError } = require('../errors');
-const perf = require('../commons/performance-logger');
-
-const BaseTestPlanRunner = require('./BaseTestPlanRunner');
-
-class AnonymousTestPlanRunner extends BaseTestPlanRunner {
-    runAnonymousTestPlan(options, branchToUse) {
-        logger.info('start to run anonymous', {
-            options: Object.assign({}, options, { token: undefined }),
-            branchToUse,
-        });
-
-        return this.prepareForTestResources(options)
-            .log('after prepareForTestResoutces')
-            .then(() => getSuite(options, branchToUse))
-            .log('after getSuite')
-            .then((suiteResult) => {
-                if (!suiteResult.tests[0] || suiteResult.tests[0].length === 0) {
-                    if (options.rerunFailedByRunId) {
-                        throw new ArgError('No failed tests found in the provided run');
-                    }
-                    if (options.passZeroTests) {
-                        return [];
-                    }
-                    throw new ArgError('No tests to run');
-                }
-                branchToUse = suiteResult.branch || branchToUse;
-                if (options.rerunFailedByRunId && !suiteResult.runName) {
-                    if (!suiteResult.runExists) {
-                        throw new ArgError('Invalid run ID - no such run.');
-                    }
-                    const isAnonymouslyNamedRun = suiteResult.runName === '';
-                    if (isAnonymouslyNamedRun) {
-                        suiteResult.runName = `rerun-${options.rerunFailedByRunId}`;
-                    }
-                }
-                const testPlanName = options.overrideExecutionName || suiteResult.runName || _.concat(options.label, options.name, options.suites).join(' ');
-                const isAnonymous = true;
-                perf.log('Right before validateConfig + runTestPlan Promise.map');
-                return Promise.map(suiteResult.tests, suiteTests => {
-                    //override result id for remote run mode and run only the first test data
-                    if (options.resultId) {
-                        const firstTest = _.first(suiteTests);
-                        firstTest.resultId = options.resultId;
-                        suiteTests = [firstTest];
-                    }
-                    return this.validateConfig(options, suiteTests)
-                        .log('right before runTestPlan')
-                        .then(() => this.runTestPlan([], suiteTests, [], options, testPlanName, null, branchToUse, isAnonymous))
-                        .log('right after runTestPlan')
-                        .tap(async res => {
-                            const isCodeMode = options.files.length > 0;
-                            await reporter.onTestPlanFinished(res.results, testPlanName, this.startTime, res.executionId, isAnonymous, isCodeMode, res.childTestResults);
-                        });
-                }).then(async results => { // array of results per execution
-                    const flattenResults = _(results).flattenDeep().value();
-                    perf.log('right before onAllTestPlansFinished');
-                    await reporter.onAllTestPlansFinished(flattenResults);
-                    perf.log('right after onAllTestPlansFinished');
-                    return flattenResults.map(res => res.results).reduce((total, cur) => Object.assign(total, cur), {});
-                });
-            });
-    }
-}
-
-async function getSuite(options, branchToUse) {
-    if (options.lightweightMode && options.lightweightMode.onlyTestIdsNoSuite && options.testId) {
-        return { tests: [options.testId.map(testId => ({ testId, testConfig: { }, resultId: utils.guid() }))] };
-    }
-    // local code test
-    if (options.files.length > 0) {
-        const { buildCodeTests } = require('./buildCodeTests');
-        let webpackConfig = {};
-        if (options.webpackConfig) {
-            const webpackConfigPath = path.join(process.cwd(), options.webpackConfig);
-            webpackConfig = require(webpackConfigPath);
-        }
-
-        return buildCodeTests(options.files, webpackConfig, { baseUrl: options.baseUrl });
-    }
-    // regular test
-    const servicesApi = require('../commons/testimServicesApi');
-    return await servicesApi.getSuiteTestList({
-        projectId: options.project,
-        labels: options.label,
-        testIds: options.testId,
-        testNames: options.name,
-        testConfigNames: options.testConfigNames,
-        suiteNames: options.suites,
-        suiteIds: options.suiteIds,
-        branch: branchToUse,
-        rerunFailedByRunId: options.rerunFailedByRunId,
-        testConfigIds: options.testConfigIds,
-    });
-}
-module.exports = AnonymousTestPlanRunner;

package/runners/BaseRunner.js

@@ -1,42 +0,0 @@
-const Promise = require('bluebird');
-const _ = require('lodash');
-const utils = require('../utils');
-const testimCustomToken = require('../commons/testimCustomToken');
-const analytics = require('../commons/testimAnalytics');
-const {ArgError} = require('../errors');
-
-class BaseRunner {
-    constructor(strategy) {
-        this.strategy = strategy;
-        this.startTime = Date.now();
-    }
-
-    analyticsExecsStart({ executionId, projectId, sessionType }) {
-        analytics.trackWithCIUser('batch-run-ci', {
-            executionId,
-            projectId,
-            sessionType,
-        });
-    }
-
-    validateConfig(options, testList) {
-        let supportedBrowsers = options.mode === 'extension' ? [
-            'edge-chromium', 'chrome',
-        ] : [
-            'ie11', 'firefox', 'chrome', 'edge', 'edge-chromium', 'safari', 'safari technology preview', 'browser', 'android', 'ios', 'iphone', 'ipad',
-        ];
-        let diff = _.difference(utils.getUniqBrowsers(options, testList), supportedBrowsers);
-
-        if (diff.length > 0) {
-            analytics.trackWithCIUser('invalid-config-run', {
-                browser: diff.join(", "),
-                mode: 'runner'
-            });
-            return Promise.reject(new ArgError(`browser type <${diff}> is not supported in ${options.mode} mode.`));
-        }
-
-        return Promise.resolve(testList);
-    }
-}
-
-module.exports = BaseRunner;

package/runners/BaseTestPlanRunner.js

@@ -1,194 +0,0 @@
-'use strict';
-
-const Promise = require('bluebird');
-const _ = require('lodash');
-const constants = require('../commons/constants');
-
-const TESTIM_RUN_STATUS = constants.testRunStatus;
-const { CLI_MODE } = constants;
-const reporter = require('../reports/reporter');
-const RealDataService = require('../commons/socket/realDataService');
-const testimServicesApi = require('../commons/testimServicesApi');
-const testimCustomToken = require('../commons/testimCustomToken');
-const BaseRunner = require('./BaseRunner');
-const TestRunStatus = require('../testRunStatus');
-const config = require('../commons/config');
-const utils = require('../utils');
-const { StopRunOnError } = require('../errors');
-const Logger = require('../commons/logger');
-const perf = require('../commons/performance-logger');
-
-const guid = utils.guid;
-const _logger = Logger.getLogger('base-test-plan-runner');
-const TDK_CHILD_RESULTS_TIMEOUT = 1000 * 60 * 5;
-
-class BaseTestPlanRunner extends BaseRunner {
-    constructor(strategy) {
-        super(strategy);
-        this.runTestPlan = Promise.method(this.runTestPlan);
-    }
-    runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, testStatus) {
-        const executionResults = {};
-        const authData = testimCustomToken.getTokenV3UserData();
-
-        const runBeforeTests = (beforeTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
-            const workerCount = 1;
-            const stopOnError = true;
-            return this.strategy.runTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
-                .then(beforeTestsResults => Object.assign(executionResults, beforeTestsResults));
-        };
-
-        const runTestPlanTests = (tests, testStatus, executionId, tpOptions, branchToUse, authData) => {
-            const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
-            const stopOnError = false;
-            perf.log('right before this.strategy.runTests');
-            return this.strategy.runTests(tests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
-                .log('right after this.strategy.runTests')
-                .then(testsResults => Object.assign(executionResults, testsResults));
-        };
-
-        const runAfterTests = (afterTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
-            const workerCount = 1;
-            const stopOnError = false;
-            return this.strategy.runTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
-                .then(afterTestsResults => Object.assign(executionResults, afterTestsResults));
-        };
-
-        function catchBeforeTestsFailed(executionId) {
-            return testStatus.markAllQueuedTests(executionId, constants.runnerTestStatus.ABORTED, 'aborted', false);
-        }
-
-        const sessionType = utils.getSessionType(tpOptions);
-        this.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
-        perf.log('right before runBeforeTests');
-        return runBeforeTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData)
-            .log('right before runTestPlanTests')
-            .then(() => runTestPlanTests(tests, testStatus, executionId, tpOptions, branchToUse, authData))
-            .log('right after runTestPlanTests')
-            .then(() => runAfterTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData))
-            .then(() => executionResults)
-            .catch(StopRunOnError, () => catchBeforeTestsFailed(executionId));
-    }
-
-    prepareForTestResources(tpOptions) {
-        if (tpOptions.mode !== CLI_MODE.APPIUM && !(this.strategy.constructor.name === 'DeviceFarmStrategy')) {
-            return Promise.resolve();
-        }
-        const apkUploaderFactory = require('../commons/apkUploader/apkUploaderFactory');
-        return apkUploaderFactory.uploadApk(tpOptions);
-    }
-
-    async initRealDataService(projectId) {
-        const realDataService = new RealDataService();
-        await realDataService.init(projectId);
-        return realDataService;
-    }
-
-    async listenToTestCreatedInFile(realDataService, projectId, runId, testStatus) {
-        const childTestResults = {};
-        realDataService.joinToTestResultsByRunId(runId, projectId);
-        const promise = new Promise(resolve => {
-            realDataService.listenToTestResultsByRunId(runId, testResult => {
-                const resultId = testResult.id;
-                if (!testStatus.getTestResult(resultId)) {
-                    const prevTestResult = childTestResults[resultId];
-                    const mergedTestResult = _.merge({}, prevTestResult, testResult, { resultId });
-                    childTestResults[resultId] = mergedTestResult;
-                    if (!prevTestResult || prevTestResult.status !== testResult.status) {
-                        const parentTestResult = testStatus.getTestResult(mergedTestResult.parentResultId) || { workerId: 1 };
-                        const workerId = parentTestResult.workerId;
-                        if ([TESTIM_RUN_STATUS.RUNNING].includes(testResult.status)) {
-                            reporter.onTestStarted(mergedTestResult, workerId);
-                        }
-                        if ([TESTIM_RUN_STATUS.COMPLETED].includes(testResult.status)) {
-                            mergedTestResult.duration = (mergedTestResult.endTime - mergedTestResult.startTime) || 0;
-                            reporter.onTestFinished(mergedTestResult, workerId);
-                        }
-                    }
-                }
-
-                const allChildTestResultCompleted = !(Object.values(childTestResults)
-                    .some(result => ['QUEUED', 'RUNNING'].includes(result.runnerStatus)));
-
-                const allParentTestResultCompleted = !(Object.values(testStatus.getAllTestResults())
-                    .some(result => ['QUEUED', 'RUNNING'].includes(result.status)));
-
-                if (allChildTestResultCompleted && allParentTestResultCompleted) {
-                    return resolve(Object.values(childTestResults));
-                }
-
-                if (allParentTestResultCompleted && !allChildTestResultCompleted) {
-                    // wait 10 sec to handle race condition when parent test result (file) finished before child test result
-                    return Promise.delay(10000)
-                        .then(() => {
-                            if (promise.isPending()) {
-                                // TODO(Benji) we are missing the child test results here.
-                                // we are resolving here with partial data - we should consider fetching it
-                                // from the server
-                                resolve(childTestResults);
-                            }
-                        });
-                }
-                return undefined;
-            });
-        });
-
-        return await promise;
-    }
-
-    async runTestPlan(beforeTests, tests, afterTests, tpOptions, testPlanName, testPlanId, branch, isAnonymous) {
-        const executionId = guid();
-        const projectId = tpOptions.project;
-        Logger.setExecutionId(executionId);
-        beforeTests.forEach(test => { test.isBeforeTestPlan = true; });
-        afterTests.forEach(test => { test.isAfterTestPlan = true; });
-        const testStatus = new TestRunStatus(_.concat(beforeTests, tests, afterTests), tpOptions, testPlanId, branch);
-
-        const configs = _(_.concat(beforeTests, tests, afterTests)).map(test => (test.overrideTestConfig && test.overrideTestConfig.name) || '').uniq().filter(Boolean)
-            .value();
-        const configName = configs && configs.length === 1 ? configs[0] : null;
-
-        const isCodeMode = tpOptions.files.length > 0;
-
-        if (isCodeMode && tpOptions.mode === constants.CLI_MODE.SELENIUM) {
-            // in selenium mode we don't need to wait for the runner and clickim to sync, so we don't need to wait for reports.
-            testStatus.setAsyncReporting(true);
-        }
-        const testListInfoPromise = tpOptions.lightweightMode && tpOptions.lightweightMode.onlyTestIdsNoSuite ?
-            { beforeTests, tests, afterTests } :
-            testStatus.executionStart(executionId, projectId, this.startTime, testPlanName);
-        let childTestResults;
-        if (isCodeMode) {
-            childTestResults = Promise.try(async () => {
-                const realDataService = await this.initRealDataService(projectId);
-                return this.listenToTestCreatedInFile(realDataService, projectId, executionId, testStatus);
-            });
-        }
-        perf.log('before testListInfoPromise');
-        const testListInfo = await testListInfoPromise;
-        if (!(tpOptions.lightweightMode && tpOptions.lightweightMode.onlyTestIdsNoSuite)) {
-            reporter.onTestPlanStarted(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, testPlanName, executionId, isAnonymous, configName, isCodeMode);
-        }
-
-        perf.log('before runTestAllPhases');
-        const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testStatus);
-        const childResults = await Promise.resolve(childTestResults)
-            .timeout(TDK_CHILD_RESULTS_TIMEOUT)
-            .catch(async () => {
-                _logger.warn('timed out waiting for child resutls on websocket. using REST fallback', { projectId, executionId });
-                const testResults = await testimServicesApi.getRealData(projectId, 'testResult', `runId=${executionId}&sort=runOrder`);
-                return _.chain((testResults && testResults.data && testResults.data.docs) || [])
-                    .groupBy('parentResultId')
-                    .omit('undefined')
-                    .values()
-                    .flatten()
-                    .value();
-            });
-        perf.log('before executionEnd');
-        await testStatus.executionEnd(executionId);
-        perf.log('after executionEnd');
-        return { results, executionId, testPlanName, configName, childTestResults: childResults };
-    }
-}
-
-module.exports = BaseTestPlanRunner;

package/runners/DeviceFarmRemoteRunner.js

@@ -1,50 +0,0 @@
-"use strict";
-
-const utils = require('../utils');
-const config = require('../commons/config');
-const testimCustomToken = require('../commons/testimCustomToken');
-const BaseRunner = require('./BaseRunner');
-const TestRunStatus = require('../testRunStatus');
-const reporter = require("../reports/reporter");
-const Logger = require('../commons/logger');
-const gridService = require('../services/gridService');
-const logger = Logger.getLogger('remote-test-runner');
-
-class DeviceFarmRemoteRunner extends BaseRunner {
-    runExecution(testList, testStatus, executionId, options, branchToUse) {
-        const projectId = options.project;
-        const sessionType = utils.getSessionType(options);
-        // report exec start analytics
-        const authData = testimCustomToken.getTokenV3UserData();
-        this.analyticsExecsStart({authData, executionId, projectId, sessionType});
-        const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || options.parallel;
-
-        return this.strategy.runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount);
-    }
-
-    runTest(executionId, testList, gridInfo, branchToUse, options) {
-        Logger.setExecutionId(executionId);
-
-        logger.info("start to remote runner", {
-            executionId: executionId,
-            options: Object.assign({}, options, {token: undefined}),
-            branchToUse: branchToUse,
-        });
-
-        const testPlanName = "remote run";
-        const testStatus = new TestRunStatus(testList, options, null, branchToUse);
-        reporter.onTestPlanStarted([], testList, [], testPlanName, executionId);
-
-        const workerId = 1;
-        const browserType = options.projectData.type; //android or ios
-        const {gridId, slotId} = gridInfo;
-        gridService.addItemToGridCache(workerId, gridId, slotId, browserType);
-        options.disableWindowAnimation = options.remoteRunObject.echoedOptions.disableWindowAnimation
-        options.baseUrl = options.remoteRunObject.echoedOptions.baseUrl
-        return this.runExecution(testList, testStatus, executionId, options, branchToUse)
-            .tap(results => reporter.onTestPlanFinished(results, testPlanName, this.startTime, executionId))
-            .then(combinedTestResults => combinedTestResults);
-    }
-}
-
-module.exports = DeviceFarmRemoteRunner;

package/runners/SchedulerRemoteRunner.js

@@ -1,47 +0,0 @@
-"use strict";
-
-const utils = require('../utils');
-const config = require('../commons/config');
-const testimCustomToken = require('../commons/testimCustomToken');
-const BaseRunner = require('./BaseRunner');
-const TestRunStatus = require('../testRunStatus');
-const reporter = require("../reports/reporter");
-const guid = require('../utils').guid;
-const Logger = require('../commons/logger');
-const logger = Logger.getLogger('remote-test-runner');
-
-class SchedulerRemoteRunner extends BaseRunner {
-    runExecution(testList, testStatus, executionId, options, branchToUse) {
-        const sessionType = utils.getSessionType(options);
-        const projectId = options.project;
-        // report exec start analytics
-        const authData = testimCustomToken.getTokenV3UserData();
-        this.analyticsExecsStart({authData, executionId, projectId, sessionType});
-        const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || options.parallel;
-
-        return this.strategy.runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount);
-    }
-
-    runTest(options, branchToUse) {
-        const executionId = options.executionId || guid();
-        const test = options.testObject;
-        Logger.setExecutionId(executionId);
-
-        logger.info("start to run test", {
-            executionId: executionId,
-            options: Object.assign({}, options, {token: undefined}),
-            branchToUse: branchToUse,
-            test: test
-        });
-
-        const testArray = [test];
-        const testPlanName = options.overrideExecutionName || "remote run";
-        const testStatus = new TestRunStatus(testArray, options, null, branchToUse);
-        reporter.onTestPlanStarted([], testArray, [], testPlanName, executionId);
-        return this.runExecution(testArray, testStatus, executionId, options, branchToUse)
-            .tap(results => reporter.onTestPlanFinished(results, testPlanName, this.startTime, executionId))
-            .then(combinedTestResults => combinedTestResults);
-    }
-}
-
-module.exports = SchedulerRemoteRunner;

package/runners/strategies/BaseStrategy.js

@@ -1,86 +0,0 @@
-'use strict';
-
-const analytics = require('../../commons/testimAnalytics');
-const { isCi } = require('../../cli/isCiRun');
-
-const calcSource = (source, user) => {
-    if (source !== 'cli' && source !== 'cli-local') {
-        return source;
-    }
-
-    if (isCi && user) {
-        return 'ci-with-user';
-    }
-
-    if (isCi) {
-        return 'ci';
-    }
-
-    if (user) {
-        return 'cli-with-user';
-    }
-
-    return source;
-};
-
-function setLightweightAnalytics(properties, lightweightMode) {
-    if (lightweightMode && lightweightMode.type) {
-        properties[`${lightweightMode.type}Mode`] = true;
-    }
-    return properties;
-}
-
-class BaseStrategy {
-    analyticsTestStart({
-        executionId, projectId, testId, resultId, companyId, companyName, projectName, companyPlanType, sessionType, source, user, lightweightMode,
-    }) {
-        const properties = setLightweightAnalytics({
-            executionId,
-            projectId,
-            testId,
-            resultId,
-            companyId,
-            companyName,
-            projectName,
-            companyPlanType,
-            sessionType,
-            source: calcSource(source, user),
-        }, lightweightMode);
-        analytics.trackWithCIUser('test-run-ci', properties);
-    }
-
-    analyticsTestEnd({
-        executionId, projectId, testId, resultId, result, companyId, companyName, projectName, companyPlanType, sessionType, source, user, lightweightMode,
-        logger,
-    }) {
-        try {
-            const properties = setLightweightAnalytics({
-                executionId,
-                projectId,
-                testId,
-                resultId,
-                companyId,
-                companyName,
-                projectName,
-                companyPlanType,
-                sessionType,
-                mockNetworkEnabled: result.wasMockNetworkActivated,
-                source: calcSource(source, user),
-            }, lightweightMode);
-
-            if (result.success) {
-                analytics.trackWithCIUser('test-run-ci-success', properties);
-                return;
-            }
-            analytics.trackWithCIUser('test-run-ci-fail', Object.assign({}, properties, { failureReason: result.failureReason }));
-        } catch (err) {
-            logger.error('failed to update test end analytics', { err });
-        }
-    }
-
-    runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount, stopOnError) {
-        throw new Error('need to implement on child');
-    }
-}
-
-module.exports = BaseStrategy;