@testim/testim-cli 3.212.0 → 3.213.0

package/commons/httpRequest.js

@@ -32,6 +32,11 @@ function getProxy() {
     return global.proxyUri;
 }
 
+const logErrorAndRethrow = (logMsg, data) => err => {
+    logger.error(logMsg, { ...data, error: err });
+    throw err;
+};
+
 function deleteMethod(url, headers, timeout) {
     return deleteFullRes(url, headers, timeout)
         .then(res => {
@@ -40,7 +45,7 @@ function deleteMethod(url, headers, timeout) {
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to delete request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to delete request', { url }));
 }
 
 function deleteFullRes(url, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
@@ -70,7 +75,7 @@ function post({
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to post request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to post request', { url }));
 }
 
 function postFullRes(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, retry) {
@@ -128,7 +133,7 @@ function postForm(url, fields, files, headers = {}, timeout = DEFAULT_REQUEST_TI
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to post request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to post request', { url }));
 }
 
 function _get(url, query, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, { isBinary = false, skipProxy = false } = {}) {
@@ -156,13 +161,16 @@ function _get(url, query, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, { isB
 function getText(url, query, headers) {
     return _get(url, query, headers)
         .then((res) => res.text)
-        .tapCatch((err) => logger.error('failed to getText request', { url, query, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to getText request', { url, query }));
 }
 
 function get(url, query, headers, timeout, options) {
     return _get(url, query, headers, timeout, options)
         .then((res) => res.body)
-        .tapCatch((err) => logger.warn('failed to get request', { url, query, error: { message: err.message, stack: err.stack } }));
+        .catch(err => {
+            logger.warn('failed to get request', { url, query, error: err });
+            throw err;
+        });
 }
 
 function getFullRes(url, query, headers, timeout) {
@@ -183,7 +191,7 @@ function head(url) {
     }
 
     return Promise.fromCallback((callback) => request.end(callback))
-        .tapCatch((err) => logger.error('failed to head request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to head request', { url }));
 }
 
 function put(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
@@ -203,7 +211,7 @@ function put(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
 
     return Promise.fromCallback((callback) => request.end(callback))
         .then((res) => res.body)
-        .tapCatch((err) => logger.error('failed to put request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to put request', { url }));
 }
 
 function download(url) {
@@ -225,7 +233,7 @@ function download(url) {
 
     return Promise.fromCallback((callback) => request.end(callback))
         .tap(() => logger.info('finished to download', { url }))
-        .tapCatch((err) => logger.error('failed to download', { error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to download', { url }));
 }
 
 module.exports = {
@@ -241,4 +249,5 @@ module.exports = {
     head: wrapWithMonitoring(head),
     download: wrapWithMonitoring(download),
     isNetworkHealthy: wrapWithMonitoring.isNetworkHealthy,
+    didNetworkConnectivityTestFail: wrapWithMonitoring.didNetworkConnectivityTestFail,
 };

package/commons/httpRequestCounters.js

@@ -1,7 +1,36 @@
 'use strit';
 
 const { sum } = require('lodash');
-const { method } = require('bluebird');
+const Bluebird = require('bluebird');
+const dns = require('dns');
+const _ = require('lodash');
+const config = require('./config');
+
+const logger = require('./logger').getLogger('http-request-counters');
+
+let networkConnectivityTestFailed = false;
+
+/** Tests network connectivity with DNS resolution (a basic test for a 7 later stack parallel but essential to most HTTP requests) */
+const testNetworkConnectivity = async () => {
+    if (config.IS_ON_PREM) {
+        return true;
+    }
+    const hostnames = ['www.google.com', 'www.facebook.com', 'www.microsoft.com', 'testim.io'];
+    try {
+        // If any of these domains resolve we consider the connectivity to be ok
+        const result = Boolean(await Bluebird.any(hostnames.map(host => dns.promises.lookup(host))));
+        if (!result) {
+            networkConnectivityTestFailed = true;
+        }
+        return result;
+    } catch (e) {
+        logger.error('network connectivity test failed');
+        networkConnectivityTestFailed = true;
+        return false;
+    }
+};
+const throttledTestNetworkConnectivity = _.throttle(testNetworkConnectivity, 10 * 1000);
+
 // we remove entries after 15 minutes, note that this isn't accurate because
 // we remove the "fail"/"success" 10 minutes after we add them (and not the "call")
 // this is fine since these values are an estimation and not an accurate representation
@@ -24,7 +53,7 @@ module.exports.makeCounters = () => {
         }, ttl);
     }
     function wrapWithMonitoring(fn, name = fn.name) {
-        return method(async function (...args) {
+        return Bluebird.method(async function (...args) {
             update(counters.call, name);
             try {
                 const result = await fn.call(this, ...args);
@@ -32,24 +61,26 @@ module.exports.makeCounters = () => {
                 return result;
             } catch (e) {
                 update(counters.fail, name);
+                if (!networkConnectivityTestFailed) {
+                    throttledTestNetworkConnectivity();
+                }
                 throw e;
             }
         });
     }
-    wrapWithMonitoring.isNetworkHealthy = function isNetworkHealthy() {
+    wrapWithMonitoring.isNetworkHealthy = async function isNetworkHealthy() {
+        if (networkConnectivityTestFailed || !(await testNetworkConnectivity())) {
+            return false;
+        }
         const allFailed = sum([...counters.fail.values()]);
-        const allSuccess = sum([...counters.success.values()]);
         const allCalls = sum([...counters.call.values()]);
         // we declare a test unhealthy network wise if
-        // 1. more than 10 requests failed and there are fewer than 100 requests
-        // 2. more than 10% of requests (out of finished requests) failed
+        // 10% or more of requests (out of finished requests) failed
         // note that the network can be unhealthy but the test would still pass
-        if (allCalls < 100) {
-            return allFailed < 10;
-        }
-        return (allSuccess + allFailed) > allFailed * 10;
+        return allFailed < allCalls * 0.1;
     };
     wrapWithMonitoring.counters = counters; // expose the counters used to the outside
     wrapWithMonitoring.isNetworkHealthy.counters = wrapWithMonitoring.counters;
+    wrapWithMonitoring.didNetworkConnectivityTestFail = () => networkConnectivityTestFailed;
     return wrapWithMonitoring;
 };

package/commons/httpRequestCounters.test.js

@@ -9,30 +9,30 @@ describe('the http request counters', () => {
     });
 
     it('Marks an always failing network as unhealthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         await rejects(fn);
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), false);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), false);
     });
 
     it('Marks an unstable network as unhealthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         const fn2 = runWithRetries(wrapWithMonitoring(() => 'hello'), 20, 1);
         await rejects(fn);
         await doesNotReject(fn2);
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), false);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), false);
     });
 
-    it('Marks a recovering network as healthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+    it('Marks a trivial amount of failed requests as healthy', async () => {
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         await rejects(fn);
         const fn2 = wrapWithMonitoring(() => 'hello');
-        await Promise.all(Array(200).fill().map(fn2));
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), true);
+        await Promise.all(Array(290).fill().map(fn2));
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), true);
     });
 
     it('Marks a healthy network as healthy', async () => {
         const fn2 = wrapWithMonitoring(() => 'hello');
         await Promise.all(Array(200).fill().map(fn2));
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), true);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), true);
     });
 });

package/npm-shrinkwrap.json

@@ -2509,9 +2509,9 @@
       },
       "dependencies": {
         "@types/node": {
-          "version": "17.0.2",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.2.tgz",
-          "integrity": "sha512-JepeIUPFDARgIs0zD/SKPgFsJEAF0X5/qO80llx59gOxFTboS9Amv3S+QfB7lqBId5sFXJ99BN0J6zFRvL9dDA=="
+          "version": "17.0.4",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.4.tgz",
+          "integrity": "sha512-6xwbrW4JJiJLgF+zNypN5wr2ykM9/jHcL7rQ8fZe2vuftggjzZeRSM4OwRc6Xk8qWjwJ99qVHo/JgOGmomWRog=="
         },
         "mkdirp": {
           "version": "0.5.5",
@@ -3501,9 +3501,9 @@
       "integrity": "sha512-GJCAeDBKfREgkBtgrYSf9hQy9kTb3helv0zGdzqhM7iAkW8FA/ZF97VQDbwFiwIT8MQLLOe5VlPZOEvZAqtUAQ=="
     },
     "electron-to-chromium": {
-      "version": "1.4.26",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.26.tgz",
-      "integrity": "sha512-cA1YwlRzO6TGp7yd3+KAqh9Tt6Z4CuuKqsAJP6uF/H5MQryjAGDhMhnY5cEXo8MaRCczpzSBhMPdqRIodkbZYw==",
+      "version": "1.4.27",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.27.tgz",
+      "integrity": "sha512-uZ95szi3zUbzRDx1zx/xnsCG+2xgZyy57pDOeaeO4r8zx5Dqe8Jv1ti8cunvBwJHVI5LzPuw8umKwZb3WKYxSQ==",
       "dev": true
     },
     "emoji-regex": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testim/testim-cli",
-  "version": "3.212.0",
+  "version": "3.213.0",
   "description": "Command line interface for running Testing on your CI",
   "author": "Oren Rubin",
   "contributors": [{

package/player/stepActions/inputFileStepAction.js

@@ -34,7 +34,10 @@ class InputFileStepAction extends StepAction {
             top: '10px',
         };
         return this.forceInputToBeVisible(target, options)
-            .tapCatch(err => logger.error('failed to set input file in Safari recovery', { err }));
+            .catch(err => {
+                logger.error('failed to set input file in Safari recovery', { err });
+                throw err;
+            });
     }
 
     uploadFilesAndForceVisibility(gridLocalFiles, target) {

package/player/stepActions/pixelValidationStepAction.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const _ = require('lodash');
 const StepAction = require('./stepAction');
 const { eyeSdkService } = require('../utils/eyeSdkService');
 const logger = require('../../commons/logger').getLogger('pixel-validation-step-action');
@@ -9,6 +10,7 @@ class PixelValidationStepAction extends StepAction {
     async performAction() {
         const { shouldUseVisualGrid, applitoolsSdkConfig: config, testResultId } = this.context;
         this.runContext = this.context.getRunContext(undefined);
+        const finalParams = (this.runContext.incomingParams && this.runContext.incomingParams.final) || {};
         const batchId = (config.batch && config.batch.id) || testResultId;
         const eyeManager = await eyeSdkService.getManager(shouldUseVisualGrid, this.context.config.applitoolsConcurrency || 5, batchId, this.runContext.applitoolsIntegrationData);
         const targetElementData = this.getTarget() || {};
@@ -16,7 +18,12 @@ class PixelValidationStepAction extends StepAction {
         try {
             const openedEye = await eyeManager.openEyes({ driver: this.driver.client, config });
             const region = (this.step.action === 'element' && targetElementData.seleniumElement) || undefined;
-            await openedEye.check({ settings: { region, fully: this.step.action === 'stitched' } });
+            const settings = { region, fully: this.step.action === 'stitched' };
+            if (finalParams.applitoolsStepSettings && _.isPlainObject(finalParams.applitoolsStepSettings)) {
+                Object.assign(settings, finalParams.applitoolsStepSettings);
+                logger.info('Applitools SDK step executed with applitoolsStepSettings parameter', { applitoolsStepSettings: finalParams.applitoolsStepSettings });
+            }
+            await openedEye.check({ settings });
             const eyesResults = await openedEye.close();
 
             result = { isApplitoolsSdkResult: true, success: true, eyesResults };

package/testRunHandler.js

@@ -18,6 +18,8 @@ const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
 const { preloadTests } = require('./commons/preloadTests');
 
 const RETRIES_ON_TIMEOUT = 3;
+const MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE = 50 * 1000; // max size, in characters, of stringified run data sent over URL params. Chosen arbitrarily, this value should be changed according to data.
+const canSendRunDataOverUrl = (runData) => JSON.stringify(runData).length < MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE;
 
 const TestRun = function (executionId, executionName, test, options, branchToUse, testRunStatus) {
     this._executionId = executionId;
@@ -175,7 +177,13 @@ TestRun.prototype.getRunRequestParams = async function () {
 
     if (this._options.lightweightMode && this._options.lightweightMode.general) {
         runRequestParams.company = this._options.company;
-        runRequestParams.runData = this.getRunData();
+        const runData = this.getRunData();
+        runRequestParams.lightweightMode.isRunDataSentInUrl = canSendRunDataOverUrl(runData);
+        if (runRequestParams.lightweightMode.isRunDataSentInUrl) {
+            runRequestParams.runData = runData;
+        } else {
+            logger.warn(`Run data is too big to be sent as a URL param (lightweight mode). Run data size: ${JSON.stringify(runData).length} (limit: ${MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE} characters)`);
+        }
         runRequestParams.isLocalRun = Boolean(this._options.useLocalChromeDriver || this._options.useChromeLauncher);
     }
 
@@ -240,7 +248,7 @@ TestRun.prototype.clearTestResult = function () {
     const mustClearPreviousStepResults = !this.isAllowReportTestResultRetries() && (this._timeoutRetryCount > 1 || this._retryCount > 1);
 
     if (this._options.lightweightMode && this._options.lightweightMode.disableResults &&
-        !mustClearPreviousStepResults) {
+            !mustClearPreviousStepResults && canSendRunDataOverUrl(runData)) {
         return Promise.resolve();
     }
 

package/workers/BaseWorker.js

@@ -8,7 +8,7 @@ const { timeoutMessages, testRunStatus, stepResult, runnerTestStatus } = require
 const logger = require('../commons/logger').getLogger('base-worker');
 const testResultService = require('../commons/socket/testResultService');
 const remoteStepService = require('../commons/socket/remoteStepService');
-const { isNetworkHealthy } = require('../commons/httpRequest');
+const { isNetworkHealthy, didNetworkConnectivityTestFail } = require('../commons/httpRequest');
 const testimServicesApi = require('../commons/testimServicesApi');
 const gridService = require('../services/gridService');
 const LambdatestService = require('../services/lambdatestService');
@@ -264,13 +264,14 @@ class BaseWorker {
                 return undefined;
             }
         };
+        const getNetworkErrorMessage = () => 'Due to network connectivity issues, Testim CLI has been unable to connect to the grid.\n' +
+                                             `Please make sure the CLI has stable access to the internet. ${didNetworkConnectivityTestFail() ? '(Internal: network connectivity test failed)' : ''}`;
 
-        const buildError = (err) => {
-            if (!isNetworkHealthy() && featureFlags.flags.errorMessageOnBadNetwork.isEnabled()) {
+        const buildError = (err, wasNetworkHealthy) => {
+            if (!wasNetworkHealthy && featureFlags.flags.errorMessageOnBadNetwork.isEnabled()) {
                 return {
                     errorType: NETWORK_ERROR,
-                    reason: 'Due to network connectivity issues, Testim CLI has been unable to connect to the grid.' +
-                        'Please make sure the CLI has stable access to the internet.',
+                    reason: getNetworkErrorMessage(),
                 };
             }
 
@@ -324,17 +325,16 @@ class BaseWorker {
         };
 
         const onRunError = async (err, testRunHandler) => {
-            if (!isNetworkHealthy() && featureFlags.flags.warnOnBadNetwork.isEnabled()) {
+            const wasNetworkHealthy = await isNetworkHealthy();
+            if (!wasNetworkHealthy && featureFlags.flags.warnOnBadNetwork.isEnabled()) {
                 // intentional, we want to log to stderr:
                 // eslint-disable-next-line no-console
-                console.warn('Due to network connectivity issues, Testim CLI has been unable to connect to the grid.');
-                // eslint-disable-next-line no-console
-                console.warn('Please make sure the CLI has stable access to the internet.');
+                console.warn(getNetworkErrorMessage());
             }
             logger.warn('error on run', { err });
 
             const projectId = this.userData && this.userData.projectId;
-            const { errorType, reason } = buildError(err);
+            const { errorType, reason } = buildError(err, wasNetworkHealthy);
             testimServicesApi.updateTestResult(projectId, this.testResultId, this.testId, {
                 status: testRunStatus.COMPLETED,
                 success: false,

package/workers/WorkerExtension.js

@@ -87,7 +87,10 @@ class WorkerExtension extends BaseWorker {
                 .then(url => {
                     reporter.onWaitToTestStart(this.id);
                     return Promise.all([
-                        driver.url(url).tap(() => { startStausDetails.driverUrlFinished = true; }).tapCatch(err => logger.error('error from driver.url', { err, testResultId, executionId, testId })),
+                        driver.url(url).tap(() => { startStausDetails.driverUrlFinished = true; }).catch(err => {
+                            logger.error('error from driver.url', { err, testResultId, executionId, testId, url, urlLength: url.length });
+                            throw err;
+                        }),
                         testRunHandler.onStarted(TEST_START_TIMEOUT_MS).tap(() => { startStausDetails.testRunHandlerStartedFinished = true; }),
                     ])
                         .timeout(TEST_START_TIMEOUT_MS, timeoutMessages.TEST_START_TIMEOUT_MSG)
@@ -110,7 +113,7 @@ class WorkerExtension extends BaseWorker {
                         .tap(testResult => {
                             driver.unregisterToClosedBrowser(onBrowserClosed);
                             if (this.lambdatestService.isLambdatestRun()) {
-                                return driver.executeJS(`lambda-status=${!testResult.success ? 'failed' : 'passed'}`).catch(() => {});
+                                return driver.executeJS(`lambda-status=${!testResult.success ? 'failed' : 'passed'}`).catch(() => { });
                             }
                             return undefined;
                         })

package/workers/workerUtils.js

@@ -40,7 +40,10 @@ module.exports.getBrowserWithRetries = ({ getBrowserOnce, testPlayerFactory, rel
         return getBrowserOnce(player)
             .then((getBrowserRes) => player || getBrowserRes)
             .timeout(singleGetBrowserDuration, timeoutMessages.GET_BROWSER_TIMEOUT_MSG)
-            .tapCatch(() => reporter.onGetBrowserFailure(workerId, projectId, ++failedAttempts))
+            .catch(err => {
+                reporter.onGetBrowserFailure(workerId, projectId, ++failedAttempts);
+                throw err;
+            })
             .tap(() => reporter.onGetBrowserSuccess(workerId, projectId))
             .catch(err => waitUntilBrowserTimeout(err, startTime, singleGetBrowserDuration, projectId, workerId, player, releaseSlotOnTestFinished));
     }, maxGetBrowserAttempts).catch(err => {