@testim/testim-cli 3.193.0 → 3.197.0

package/runOptions.js

@@ -72,11 +72,9 @@ const printUsage = () => {
         return line.includes('--player-path') || line.includes('--player-require-path');
     }
 
-    function isTestObject(line) {
+    function isScheduler(line) {
         return (
-            line.includes('--testObject') ||
             line.includes('--executionId') ||
-            line.includes('--testObject2') ||
             line.includes('--source') ||
             line.includes('--resultId') ||
             line.includes('--remoteRunId') ||
@@ -84,15 +82,6 @@ const printUsage = () => {
         );
     }
 
-    function isAwsDF(line) {
-        return (
-            line.includes('--aws-project-arn') ||
-            line.includes('--aws-region') ||
-            line.includes('--aws-access-key-id') ||
-            line.includes('--aws-secret-access-key')
-        );
-    }
-
     function isWebdriverTimeout(line) {
         return (
             line.includes('--get-browser-timeout') ||
@@ -116,6 +105,10 @@ const printUsage = () => {
         return line.includes('--save-rca-locally');
     }
 
+    function isExitCodeIgnoreFailingTests(line) {
+        return line.includes('--exit-code-ignore-failing-tests');
+    }
+
     program.help((txt) => {
         const lines = txt.split('\n');
         return lines
@@ -125,12 +118,12 @@ const printUsage = () => {
                     !isParamsJsonOption(ln) &&
                     !isDefaultHelpLine(ln) &&
                     !isPlayerOption(ln) &&
-                    !isTestObject(ln) &&
+                    !isScheduler(ln) &&
                     !isMonitorPerformance(ln) &&
                     !isUserId(ln) &&
-                    !isAwsDF(ln) &&
                     !isWebdriverTimeout(ln) &&
-                    !isSaveRCALocally(ln)
+                    !isSaveRCALocally(ln) &&
+                    !isExitCodeIgnoreFailingTests(ln)
             )
             .join('\n');
     });
@@ -304,9 +297,6 @@ program
     .option('--disable-sockets', 'Disable CLI sockets', false)
 
     // Remote run options
-    .option('--testObject [test-object]', '', '')
-    .option('--remoteRunObject [remote-run-object]', '', '')
-    .option('--testObject2 [test-object]', '', '')
     .option('--executionId [execution-id]', '', '')
     .option('--remoteRunId [remote-run-id]', '', '')
     .option('--schedulerId [scheduler-id]', '', '')
@@ -319,17 +309,6 @@ program
     .option('--agent-port [agent-port]', 'set agent port', Number, 42543)
     .option('--agent-bind [agent-host-bind]', 'set agent host bind', '127.0.0.1')
 
-    // Mobile options
-    .option('--apk-location [apk-local-location]', 'application apk location')
-    .option('--ipa-location [ipa-local-location]', 'application ipa location')
-    .option('--apk-timeout [apk-upload-timeout]', 'upload apk timeout in milliseconds', 15 * 60 * 1000)
-    .option('--ipa-timeout [ipa-upload-timeout]', 'upload ipa timeout in milliseconds', 15 * 60 * 1000)
-    .option('--disable-window-animation [disable-window-animation]', 'disable window animation by setting Appium capability', false)
-
-    .option('--device-udid [device-udid]', 'device udid')
-    .option('--device-name [device-name]', 'device name', 'mobile')
-    .option('--vendor-app-id [vendor-app-id]', 'use existing application id instead of upload new app')
-
     .option('--chrome-extra-prefs [chrome-extra-prefs]', 'add extra chrome preferences', '')
     .option('--chrome-extra-args [chrome-extra-args]', 'add extra chrome arguments separated by \',\'', '')
     .option('--chrome-block-location [chrome-block-location]', 'block chrome geolocation', false)
@@ -342,17 +321,13 @@ program
     .option('--version [version]', 'print the current version of the Testim CLI and exit', false)
     .option('--monitor-performance', 'collect test playback performance data')
 
-    // AWS DF
-    .option('--aws-project-arn [projectArn]', 'projectArn', '')
-    .option('--aws-region [region]', 'region', 'us-west-2')
-    .option('--aws-access-key-id [accessKeyId]', 'accessKeyId', '')
-    .option('--aws-secret-access-key [secretAccessKey]', 'secretAccessKey', '')
-
     .option('--high-speed', 'run in high speed mode')
     .option('--lightweight-mode [settings]', 'run lightweight mode')
     .option('--create-prefeched-data [location]', 'prefech data into local cache file')
     .option('--use-prefeched-data [location]', 'use prefeched data from local cache file, and force using only cached data')
     .option('--save-rca-locally [path]', 'save root cause analysis assets locally')
+
+    .option('--exit-code-ignore-failing-tests', 'dont return non zero exit code when tests fail. non zero exit code will mean a real error occurred')
     .parse(process.argv);
 
 module.exports = {
@@ -814,7 +789,7 @@ module.exports = {
             return Promise.reject(new ArgError('parallel could not be a negative number or not number, --parallel <number-of-tests>'));
         }
 
-        if ([CLI_MODE.EXTENSION, CLI_MODE.SELENIUM, CLI_MODE.APPIUM, CLI_MODE.DEVICE_FARM_APPIUM].indexOf(program.mode) === -1) {
+        if ([CLI_MODE.EXTENSION, CLI_MODE.SELENIUM].indexOf(program.mode) === -1) {
             return Promise.reject(new ArgError(`runner mode <${program.mode}> is not supported`));
         }
 
@@ -825,38 +800,11 @@ module.exports = {
         if (
             !program.browser &&
             !isTestConfigSpecified &&
-            !isTestPlanSpecified &&
-            !program.testObject2
+            !isTestPlanSpecified
         ) {
             program.browser = 'chrome';
         }
 
-        if (program.testObject) {
-            try {
-                program.testObject = JSON.parse(program.testObject);
-            } catch (err) {
-                return Promise.reject(new ArgError(`failed to parse testObject string error: ${err.message}`));
-            }
-        }
-
-        if (program.testObject2) {
-            try {
-                const obj = decodeURIComponent(program.testObject2);
-                program.testObject = JSON.parse(obj);
-            } catch (err) {
-                return Promise.reject(new ArgError(`failed to parse testObject string error: ${err.message}`));
-            }
-        }
-
-        if (program.remoteRunObject) {
-            try {
-                const obj = decodeURIComponent(program.remoteRunObject);
-                program.remoteRunObject = JSON.parse(obj);
-            } catch (err) {
-                return Promise.reject(new ArgError(`failed to parse remoteRunObject string error: ${err.message}`));
-            }
-        }
-
         if (program.testPlan && program.testPlan.length === 0 && program.testPlanId && program.testPlanId.length === 0) {
             if (
                 typeof program.host !== 'string' &&
@@ -950,9 +898,6 @@ module.exports = {
         } else {
             program.files = [];
         }
-        if (program.apkLocation && program.ipaLocation) {
-            return Promise.reject(new ArgError('cannot set --apk-location and --ipa-location. please choose the one compatible with your project type'));
-        }
 
         if (program.setRetention && !_.inRange(_.parseInt(program.setRetention), 1, 11)) {
             return Promise.reject(new ArgError('Please provide the number of days that the test results will be retained for (--set-retention must be a whole number between 1 to 10)'));
@@ -1119,7 +1064,6 @@ module.exports = {
             testobjectSauce: program.testobjectSauce,
             gridUsername: program.gridUsername,
             gridPassword: program.gridPassword,
-            vendorAppId: program.vendorAppId,
             overrideExecutionName: program.overrideExecutionName,
 
             tmsSuppressReporting: Boolean(program.suppressTmsReporting) || Boolean(program.tmsSuppressReporting),
@@ -1186,9 +1130,7 @@ module.exports = {
             codeCoverageInclude: program.codeCoverageInclude,
 
             // Remote run options
-            testObject: program.testObject,
             executionId: program.executionId,
-            remoteRunObject: program.remoteRunObject,
             remoteRunId: program.remoteRunId,
             schedulerId: program.schedulerId,
             source: program.source,
@@ -1200,19 +1142,6 @@ module.exports = {
             w3cCapabilities: program.w3cCapabilities,
             oldCapabilities: program.oldCapabilities,
 
-            //APK options
-            apkLocation: program.apkLocation,
-            apkTimeout: program.apkTimeout,
-
-            // IPA options
-            ipaLocation: program.ipaLocation,
-            ipaTimeout: program.ipaTimeout,
-
-            disableWindowAnimation: program.disableWindowAnimation,
-
-            deviceUdid: program.deviceUdid,
-            deviceName: program.deviceName,
-
             chromeBlockLocation: program.chromeBlockLocation,
             chromeUserDataDir: program.chromeUserDataDir,
             retries: program.retries,
@@ -1226,16 +1155,11 @@ module.exports = {
 
             user: program.user,
 
-            awsProjectArn: program.awsProjectArn,
-            awsRegion: program.awsRegion,
-
-            awsAccessKeyId: program.awsAccessKeyId,
-            awsSecretAccessKey: program.awsSecretAccessKey,
-
             lightweightMode: program.lightweightMode,
             createPrefechedData: program.createPrefechedData,
 
             saveRCALocally: program.saveRcaLocally,
+            exitCodeIgnoreFailingTests: program.exitCodeIgnoreFailingTests,
 
             disableSockets: program.disableSockets,
         });

package/runner.js

@@ -1,7 +1,7 @@
 'use strict';
 
 /* eslint-disable no-console */
-const { CLI_MODE, gridTypes } = require('./commons/constants');
+const { CLI_MODE } = require('./commons/constants');
 const Promise = require('bluebird');
 const _ = require('lodash');
 const { EDITOR_URL } = require('./commons/config');
@@ -15,16 +15,13 @@ const npmDriver = require('./testimNpmDriver.js');
 const analytics = require('./commons/testimAnalytics');
 const branchService = require('./services/branchService');
 const gridService = require('./services/gridService');
-const strategySelector = require('./runners/strategies/Strategy');
 const { ArgError, QuotaDepletedError } = require('./errors');
 const featureFlags = require('./commons/featureFlags');
 const perf = require('./commons/performance-logger');
 const { prepareMockNetwork, initializeUserWithAuth } = require('./commons/prepareRunner');
-const { hasTestPlanFlag } = require('./utils');
 
 const FREE_PLAN_MINIMUM_BROWSER_TIMEOUT = 30 * 60 * 1000;
 
-const AnonymousTestPlanRunner = require('./runners/AnonymousTestPlanRunner');
 const TestPlanRunner = require('./runners/TestPlanRunner');
 const labFeaturesService = require('./services/labFeaturesService');
 const featureAvailabilityService = require('./commons/featureAvailabilityService');
@@ -85,17 +82,6 @@ function validateCliAccount(options) {
     });
 }
 
-function validateRemoteRunAccount(options) {
-    return validateProjectQuotaNotDepleted(options)
-        .catch(err => {
-            if (err instanceof QuotaDepletedError) {
-                return Promise.reject(err);
-            }
-            logger.error('could not validate remote run account', { err });
-            return undefined;
-        });
-}
-
 function analyticsIdentify(projectId) {
     const authData = testimCustomToken.getTokenV3UserData();
     return analytics.identify({
@@ -138,7 +124,7 @@ function setCompany(options, company) {
     const { onprem, id, storageBaseUrl, storageType, name, activePlan = {} } = company;
     if (onprem) {
         const { mode, extensionPath, ext, playerPath } = options;
-        if ([CLI_MODE.SELENIUM, CLI_MODE.APPIUM].includes(mode) && !playerPath) {
+        if ([CLI_MODE.SELENIUM].includes(mode) && !playerPath) {
             return Promise.reject(new ArgError('in selenium on prem mode --player-path must be provided'));
         }
         if (mode === 'extension' && !extensionPath && !ext) {
@@ -214,32 +200,13 @@ async function setMockNetworkRules(options) {
 
 function runRunner(options, customExtensionLocalLocation) {
     perf.log('in runner.js runRunner');
-    const branchToUse = branchService.getCurrentBranch();
-
-    const selectedStrategy = strategySelector(options, customExtensionLocalLocation);
-    const {
-        remoteRunObject, testObject, executionId, project, remoteRunId, useLocalChromeDriver, useChromeLauncher,
-    } = options;
 
-    if (testObject && executionId) {
-        const SchedulerRemoteRunner = require('./runners/SchedulerRemoteRunner');
-        const schedulerRemoteRunner = new SchedulerRemoteRunner(selectedStrategy);
-        return validateRemoteRunAccount(options)
-            .then(() => schedulerRemoteRunner.runTest(options, branchToUse));
-    }
+    const { project, remoteRunId, useLocalChromeDriver, useChromeLauncher } = options;
 
     if (!remoteRunId) {
         options.source = (useLocalChromeDriver || useChromeLauncher) ? 'cli-local' : 'cli';
     }
 
-    if (remoteRunObject && executionId) {
-        const { testList, gridInfo } = remoteRunObject;
-        const DeviceFarmRemoteRunner = require('./runners/DeviceFarmRemoteRunner');
-        const deviceFarmRemoteRunner = new DeviceFarmRemoteRunner(selectedStrategy);
-        return validateRemoteRunAccount(options)
-            .then(() => deviceFarmRemoteRunner.runTest(executionId, testList, gridInfo, branchToUse, options));
-    }
-
     npmDriver.checkNpmVersion();
     perf.log('in runner.js after checkNpmVersion');
 
@@ -247,15 +214,7 @@ function runRunner(options, customExtensionLocalLocation) {
         .log('in runRunner before tunnel.connect')
         .then(() => tunnel.connect(options, testimCustomToken.getTokenV3UserData()))
         .log('in runRunner after tunnel.connect')
-        .then(() => {
-            if (hasTestPlanFlag(options)) {
-                const testPlanRunner = new TestPlanRunner(selectedStrategy);
-                return testPlanRunner.runTestPlans(options, branchToUse);
-            }
-            const anonymousTestPlanRunner = new AnonymousTestPlanRunner(selectedStrategy);
-            perf.log('right before runAnonymousTestPlan');
-            return anonymousTestPlanRunner.runAnonymousTestPlan(options, branchToUse);
-        })
+        .then(() => new TestPlanRunner(customExtensionLocalLocation).run(options))
         .log('before tunnel.disconnect')
         .tap(() => tunnel.disconnect(options))
         .tap(() => gridService.keepAlive.end(project))
@@ -309,10 +268,14 @@ async function init(options) {
         await labFeaturesService.loadLabFeatures(projectById.id, companyByProjectId.activePlan);
     }
 
-    if (options.lightweightMode && options.lightweightMode.type === 'highSpeed' && !labFeaturesService.isFeatureAvailableForProject('highSpeedMode')) {
+    if (options.lightweightMode && options.lightweightMode.type === 'highSpeed' && (!labFeaturesService.isFeatureAvailableForProject('highSpeedMode') || options.company.planType === 'free')) {
         delete options.lightweightMode;
     }
 
+    if (options.lightweightMode && options.lightweightMode.type === 'highSpeed') {
+        console.log('High-speed mode will ignore step delays. Test artifacts like screenshots and logs will only be saved for failed runs. For more information see our docs: https://help.testim.io/docs/high-speed-mode');
+    }
+
     gridService.keepAlive.start(project);
     analyticsIdentify(project);
     await setMockNetworkRules(options);

package/runners/{strategies/LocalStrategy.js → ParallelWorkerManager.js}

@@ -1,53 +1,43 @@
 'use strict';
 
-const Promise = require('bluebird');
-
-const utils = require('../../utils');
-const logger = require('../../commons/logger').getLogger('local-strategy');
-const { CLI_MODE } = require('../../commons/constants');
-const BaseStrategy = require('./BaseStrategy');
-const config = require('../../commons/config');
-const ExecutionQueue = require('../../executionQueue');
-
-const testimCustomToken = require('../../commons/testimCustomToken');
-const testResultService = require('../../commons/socket/testResultService');
-const labFeaturesService = require('../../services/labFeaturesService');
-const perf = require('../../commons/performance-logger');
-const { StopRunOnError } = require('../../errors');
-
-require('../../player/webdriver'); // preload
-
-class LocalStrategy extends BaseStrategy {
+const utils = require('../utils');
+const logger = require('../commons/logger').getLogger('parallel-worker-manager');
+const { CLI_MODE } = require('../commons/constants');
+const analyticsService = require('../services/analyticsService');
+const config = require('../commons/config');
+const ExecutionQueue = require('../executionQueue');
+
+const testimCustomToken = require('../commons/testimCustomToken');
+const testimServicesApi = require('../commons/testimServicesApi');
+const labFeaturesService = require('../services/labFeaturesService');
+const perf = require('../commons/performance-logger');
+const { StopRunOnError } = require('../errors');
+
+require('../player/webdriver'); // preload
+
+class ParallelWorkerManager {
     constructor(customExtensionLocalLocation) {
-        super();
         this.customExtensionLocalLocation = customExtensionLocalLocation;
     }
 
-    startWorker(worker, options, onTestStarted, onTestCompleted, executionId, onGridSlot, onTestIgnored) {
-        throw new Error('need to implement');
-    }
-
     getWorkerType(mode) {
         switch (mode) {
             case CLI_MODE.SELENIUM:
-                return require('../../workers/WorkerSelenium');
-            case CLI_MODE.APPIUM:
-            case CLI_MODE.DEVICE_FARM_APPIUM:
-                return require('../../workers/WorkerAppium');
+                return require('../workers/WorkerSelenium');
             default:
                 if (labFeaturesService.isFeatureAvailableForProject('useSameBrowserForMultiTests')) {
-                    return require('../../workers/WorkerExtensionSingleBrowser');
+                    return require('../workers/WorkerExtensionSingleBrowser');
                 }
-                return require('../../workers/WorkerExtension');
+                return require('../workers/WorkerExtension');
         }
     }
 
-    createWorkers(count, queue, mode) {
+    createWorkers(count, queue, mode, ...args) {
         const Worker = this.getWorkerType(mode);
         const createWorker = () => {
             try {
                 perf.log('before new Worker', mode);
-                return new Worker(queue, this.customExtensionLocalLocation);
+                return new Worker(queue, ...args);
             } finally {
                 perf.log('after new Worker', mode);
             }
@@ -56,9 +46,9 @@ class LocalStrategy extends BaseStrategy {
         return Array.from(new Array(count), createWorker);
     }
 
-    runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount, stopOnError) {
+    async runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount, stopOnError) {
         if (testList && testList.length === 0) {
-            return Promise.resolve();
+            return undefined;
         }
 
         const runAndWaitToComplete = token => new Promise((resolve, reject) => {
@@ -72,13 +62,13 @@ class LocalStrategy extends BaseStrategy {
             const companyName = options.company && options.company.name;
             const source = options.source || 'cli';
             const user = options.user;
-            const companyPlanType = options.company && options.company.planType;
+            const companyPlan = options.company && options.company.planType;
             const projectName = options.projectData && options.projectData.name;
             const lightweightMode = options.lightweightMode;
             const sessionType = utils.getSessionType(options);
 
             const onTestStarted = (wid, testId, resultId, isRerun, testRetryKey) => {
-                this.analyticsTestStart({
+                analyticsService.analyticsTestStart({
                     authData,
                     executionId,
                     projectId,
@@ -87,7 +77,7 @@ class LocalStrategy extends BaseStrategy {
                     companyId,
                     companyName,
                     projectName,
-                    companyPlanType,
+                    companyPlan,
                     sessionType,
                     source,
                     user,
@@ -95,11 +85,8 @@ class LocalStrategy extends BaseStrategy {
                 });
                 return testStatus.testStartAndReport(wid, executionId, resultId, isRerun, testRetryKey);
             };
-            const onTestCompleted = (wid, testId, testResult, sessionId, isRerun) => testStatus.testEndAndReport(wid, testResult, executionId, sessionId, isRerun).then(() => {
-                const isLocalRun = Boolean(options.useLocalChromeDriver || options.useChromeLauncher);
-                if (lightweightMode && lightweightMode.disableResults && isLocalRun) {
-                    return undefined;
-                }
+
+            const onTestCompleted = async (wid, testId, testResult, sessionId, isRerun) => {
                 const update = {};
                 if (lightweightMode && lightweightMode.onlyTestIdsNoSuite) {
                     update.show = true;
@@ -135,18 +122,15 @@ class LocalStrategy extends BaseStrategy {
                 } else if (options.saucelabs) {
                     update.gridName = 'saucelabs-from-options';
                 }
-                if (Object.keys(update).length > 0 && testId && testResult.resultId) {
-                    return testResultService.updateTestResult(projectId, testResult.resultId, testId, update).catch(err => {
-                        logger.error('failed to update test result additional info (logs and grid-info)', { err });
-                    });
-                }
-                return undefined;
-            }).then(() => {
+
+                await testStatus.testEndAndReport(wid, testResult, executionId, sessionId, isRerun, update)
+                    .catch(err => logger.error('testEndAndReport threw an error', { err }));
+
                 if (isRerun) {
                     return undefined;
                 }
                 combinedTestResults[testResult.resultId] = testResult;
-                this.analyticsTestEnd({
+                analyticsService.analyticsTestEnd({
                     authData,
                     executionId,
                     projectId,
@@ -156,23 +140,24 @@ class LocalStrategy extends BaseStrategy {
                     companyId,
                     companyName,
                     projectName,
-                    companyPlanType,
+                    companyPlan,
                     sessionType,
                     source,
                     user,
                     lightweightMode,
+                    logger,
                 });
                 if (stopOnError && !testResult.success) {
                     reject(new StopRunOnError());
-                    return Promise.reject(new StopRunOnError());
+                    throw new StopRunOnError();
                 }
                 const completedTests = Object.keys(combinedTestResults).length;
                 if (completedTests === testCount) {
                     resolve(combinedTestResults);
                     return undefined;
                 }
-                return Promise.resolve();
-            });
+                return undefined;
+            };
 
             const onTestIgnored = (wid, testResult) => {
                 combinedTestResults[testResult.resultId] = testResult;
@@ -182,6 +167,7 @@ class LocalStrategy extends BaseStrategy {
                     resolve(combinedTestResults);
                 }
             };
+
             const onGridSlot = (executionId, resultId, gridInfo) => testStatus.onGridSlot(executionId, resultId, gridInfo);
 
             options.userData = {
@@ -195,25 +181,33 @@ class LocalStrategy extends BaseStrategy {
                 servicesUrl: config.SERVICES_HOST,
             };
             perf.log('in localStrategy before createWorker');
-            this.createWorkers(workerCount, executionQueue, options.mode)
+            this.createWorkers(workerCount, executionQueue, options.mode, options, this.customExtensionLocalLocation, executionId, onTestStarted, onTestCompleted, onGridSlot, onTestIgnored)
                 .forEach((worker, index) => {
-                    perf.log('before schedule startWorker after createWorker');
+                    perf.log('before schedule worker.run after createWorkers');
                     schedule(() => {
-                        this.startWorker(worker, options, onTestStarted, onTestCompleted, executionId, onGridSlot, onTestIgnored);
+                        perf.log('right before worker.run');
+                        worker.run();
                     }, index);
                 });
-            function schedule(fn, index) {
-                if (index === 0) {
-                    fn();
-                } else {
-                    setTimeout(fn, index * config.START_WORKER_DELAY_MS);
-                }
-            }
         });
 
-        return testimCustomToken.getCustomTokenV3()
-            .then(token => runAndWaitToComplete(token));
+        try {
+            const token = await testimCustomToken.getCustomTokenV3();
+            return await runAndWaitToComplete(token);
+        } catch (err) {
+            logger.error('failed running parallel workers', { executionId, err });
+            throw err;
+        }
+    }
+}
+
+
+function schedule(fn, index) {
+    if (index === 0) {
+        fn();
+    } else {
+        setTimeout(fn, index * config.START_WORKER_DELAY_MS);
     }
 }
 
-module.exports = LocalStrategy;
+module.exports = ParallelWorkerManager;