@testim/testim-cli 3.207.0 → 3.213.0

package/agent/routers/codim/service.js

@@ -36,30 +36,45 @@ async function getLocalLocators() {
 
 async function findTests(folder = process.cwd()) {
     const testFolder = await findTestFolder(folder);
-    // note - withFileTypes requires Node 10.x so we make this in two calls.
-    const filesWithoutStat = await fs.readdirAsync(testFolder);
-    const filesWithStat = await Promise.all(filesWithoutStat.map(name => Promise.props({ stat: fs.statAsync(name).catch(e => {}), name })));
+    const filesWithStat = await fs.promises.readdir(testFolder, { withFileTypes: true });
 
     // things we know are not tests but end in js
     const excluded = ['webpack.config.js', 'tsconfig.js', '.DS_Store', 'functions.js'];
     const excludedFileTypes = ['.html', '.json'];
-    return filesWithStat.filter(x => !excluded.includes(x.name))
-        .filter(x => !excludedFileTypes.some(type => x.name.endsWith(type)))
-        .filter(x => x.stat && x.stat.isFile())
-        .filter(x => !x.name.startsWith('.'))
+    return filesWithStat
+        .filter(x =>
+            !excluded.includes(x.name) &&
+            !excludedFileTypes.some(type => x.name.endsWith(type)) &&
+            x.isFile() &&
+            !x.name.startsWith('.'),
+        )
         .map(x => x.name);
 }
 
+/**
+ * @param {Record<string, object | Promise<object>>} propsObject
+ * @returns {Promise<Record<string, object>>}
+ * */
+async function promiseFromProps(propsObject) {
+    const entries = Object.entries(propsObject);
+    const values = entries.map(([, value]) => value);
+    const resolvedValues = await Promise.all(values);
+    for (let i = 0; i < resolvedValues.length; i++) {
+        entries[i][1] = resolvedValues[i];
+    }
+    return Object.fromEntries(entries);
+}
+
 async function getLocalLocatorContents(locators, full = false, originFolder = process.cwd()) {
     const props = {};
     if (full) {
         const folder = await findTestFolder(originFolder);
         for (const key of Object.values(locators)) {
-            props[key] = fs.readFileAsync(path.join(folder, 'locators', `locator.${key}.json`)).then(JSON.parse);
+            props[key] = fs.promises.readFile(path.join(folder, 'locators', `locator.${key}.json`)).then(JSON.parse);
         }
     }
     try {
-        const contents = await Promise.props(props);
+        const contents = await promiseFromProps(props);
         return contents;
     } catch (e) {
         console.error(e);

package/commons/httpRequest.js

@@ -32,6 +32,11 @@ function getProxy() {
     return global.proxyUri;
 }
 
+const logErrorAndRethrow = (logMsg, data) => err => {
+    logger.error(logMsg, { ...data, error: err });
+    throw err;
+};
+
 function deleteMethod(url, headers, timeout) {
     return deleteFullRes(url, headers, timeout)
         .then(res => {
@@ -40,7 +45,7 @@ function deleteMethod(url, headers, timeout) {
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to delete request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to delete request', { url }));
 }
 
 function deleteFullRes(url, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
@@ -70,7 +75,7 @@ function post({
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to post request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to post request', { url }));
 }
 
 function postFullRes(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, retry) {
@@ -128,7 +133,7 @@ function postForm(url, fields, files, headers = {}, timeout = DEFAULT_REQUEST_TI
             }
             return res.body;
         })
-        .tapCatch((err) => logger.error('failed to post request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to post request', { url }));
 }
 
 function _get(url, query, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, { isBinary = false, skipProxy = false } = {}) {
@@ -156,13 +161,16 @@ function _get(url, query, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT, { isB
 function getText(url, query, headers) {
     return _get(url, query, headers)
         .then((res) => res.text)
-        .tapCatch((err) => logger.error('failed to getText request', { url, query, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to getText request', { url, query }));
 }
 
 function get(url, query, headers, timeout, options) {
     return _get(url, query, headers, timeout, options)
         .then((res) => res.body)
-        .tapCatch((err) => logger.warn('failed to get request', { url, query, error: { message: err.message, stack: err.stack } }));
+        .catch(err => {
+            logger.warn('failed to get request', { url, query, error: err });
+            throw err;
+        });
 }
 
 function getFullRes(url, query, headers, timeout) {
@@ -183,7 +191,7 @@ function head(url) {
     }
 
     return Promise.fromCallback((callback) => request.end(callback))
-        .tapCatch((err) => logger.error('failed to head request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to head request', { url }));
 }
 
 function put(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
@@ -203,7 +211,7 @@ function put(url, body, headers = {}, timeout = DEFAULT_REQUEST_TIMEOUT) {
 
     return Promise.fromCallback((callback) => request.end(callback))
         .then((res) => res.body)
-        .tapCatch((err) => logger.error('failed to put request', { url, error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to put request', { url }));
 }
 
 function download(url) {
@@ -225,7 +233,7 @@ function download(url) {
 
     return Promise.fromCallback((callback) => request.end(callback))
         .tap(() => logger.info('finished to download', { url }))
-        .tapCatch((err) => logger.error('failed to download', { error: { message: err.message, stack: err.stack } }));
+        .catch(logErrorAndRethrow('failed to download', { url }));
 }
 
 module.exports = {
@@ -241,4 +249,5 @@ module.exports = {
     head: wrapWithMonitoring(head),
     download: wrapWithMonitoring(download),
     isNetworkHealthy: wrapWithMonitoring.isNetworkHealthy,
+    didNetworkConnectivityTestFail: wrapWithMonitoring.didNetworkConnectivityTestFail,
 };

package/commons/httpRequestCounters.js

@@ -1,7 +1,36 @@
 'use strit';
 
 const { sum } = require('lodash');
-const { method } = require('bluebird');
+const Bluebird = require('bluebird');
+const dns = require('dns');
+const _ = require('lodash');
+const config = require('./config');
+
+const logger = require('./logger').getLogger('http-request-counters');
+
+let networkConnectivityTestFailed = false;
+
+/** Tests network connectivity with DNS resolution (a basic test for a 7 later stack parallel but essential to most HTTP requests) */
+const testNetworkConnectivity = async () => {
+    if (config.IS_ON_PREM) {
+        return true;
+    }
+    const hostnames = ['www.google.com', 'www.facebook.com', 'www.microsoft.com', 'testim.io'];
+    try {
+        // If any of these domains resolve we consider the connectivity to be ok
+        const result = Boolean(await Bluebird.any(hostnames.map(host => dns.promises.lookup(host))));
+        if (!result) {
+            networkConnectivityTestFailed = true;
+        }
+        return result;
+    } catch (e) {
+        logger.error('network connectivity test failed');
+        networkConnectivityTestFailed = true;
+        return false;
+    }
+};
+const throttledTestNetworkConnectivity = _.throttle(testNetworkConnectivity, 10 * 1000);
+
 // we remove entries after 15 minutes, note that this isn't accurate because
 // we remove the "fail"/"success" 10 minutes after we add them (and not the "call")
 // this is fine since these values are an estimation and not an accurate representation
@@ -24,7 +53,7 @@ module.exports.makeCounters = () => {
         }, ttl);
     }
     function wrapWithMonitoring(fn, name = fn.name) {
-        return method(async function (...args) {
+        return Bluebird.method(async function (...args) {
             update(counters.call, name);
             try {
                 const result = await fn.call(this, ...args);
@@ -32,24 +61,26 @@ module.exports.makeCounters = () => {
                 return result;
             } catch (e) {
                 update(counters.fail, name);
+                if (!networkConnectivityTestFailed) {
+                    throttledTestNetworkConnectivity();
+                }
                 throw e;
             }
         });
     }
-    wrapWithMonitoring.isNetworkHealthy = function isNetworkHealthy() {
+    wrapWithMonitoring.isNetworkHealthy = async function isNetworkHealthy() {
+        if (networkConnectivityTestFailed || !(await testNetworkConnectivity())) {
+            return false;
+        }
         const allFailed = sum([...counters.fail.values()]);
-        const allSuccess = sum([...counters.success.values()]);
         const allCalls = sum([...counters.call.values()]);
         // we declare a test unhealthy network wise if
-        // 1. more than 10 requests failed and there are fewer than 100 requests
-        // 2. more than 10% of requests (out of finished requests) failed
+        // 10% or more of requests (out of finished requests) failed
         // note that the network can be unhealthy but the test would still pass
-        if (allCalls < 100) {
-            return allFailed < 10;
-        }
-        return (allSuccess + allFailed) > allFailed * 10;
+        return allFailed < allCalls * 0.1;
     };
     wrapWithMonitoring.counters = counters; // expose the counters used to the outside
     wrapWithMonitoring.isNetworkHealthy.counters = wrapWithMonitoring.counters;
+    wrapWithMonitoring.didNetworkConnectivityTestFail = () => networkConnectivityTestFailed;
     return wrapWithMonitoring;
 };

package/commons/httpRequestCounters.test.js

@@ -9,30 +9,30 @@ describe('the http request counters', () => {
     });
 
     it('Marks an always failing network as unhealthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         await rejects(fn);
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), false);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), false);
     });
 
     it('Marks an unstable network as unhealthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         const fn2 = runWithRetries(wrapWithMonitoring(() => 'hello'), 20, 1);
         await rejects(fn);
         await doesNotReject(fn2);
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), false);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), false);
     });
 
-    it('Marks a recovering network as healthy', async () => {
-        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 20, 1);
+    it('Marks a trivial amount of failed requests as healthy', async () => {
+        const fn = runWithRetries(wrapWithMonitoring(() => { throw new Error('bad network'); }), 30, 1);
         await rejects(fn);
         const fn2 = wrapWithMonitoring(() => 'hello');
-        await Promise.all(Array(200).fill().map(fn2));
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), true);
+        await Promise.all(Array(290).fill().map(fn2));
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), true);
     });
 
     it('Marks a healthy network as healthy', async () => {
         const fn2 = wrapWithMonitoring(() => 'hello');
         await Promise.all(Array(200).fill().map(fn2));
-        strictEqual(wrapWithMonitoring.isNetworkHealthy(), true);
+        strictEqual(await wrapWithMonitoring.isNetworkHealthy(), true);
     });
 });

package/commons/testimNgrok.js

@@ -8,6 +8,7 @@ const logger = require('./logger').getLogger('testimNgrok');
 const WHITELISTED_TUNNEL_DOMAIN_SUFFIX = '.whitelisted-ngrok.testim.io';
 
 let ngrokTunnelUrl = '';
+let statsTimeout;
 
 const connectTunnel = async (options, authData) => {
     if (!authData.ngrokToken) {
@@ -28,6 +29,9 @@ const connectTunnel = async (options, authData) => {
     if (options.tunnelHostHeader) {
         connectOpt.host_header = options.tunnelHostHeader;
     }
+    if (options.tunnelRegion) {
+        connectOpt.region = options.tunnelRegion;
+    }
 
     const ngrok = await lazyRequire('ngrok');
     const url = await ngrok.connect(connectOpt);
@@ -41,14 +45,39 @@ const connectTunnel = async (options, authData) => {
         ngrokTunnelUrl = url;
     }
 
+    if (options.tunnelDiagnostics) {
+        collectNgrokStats();
+    }
     options.baseUrl = ngrokTunnelUrl;
 };
 
-const disconnectTunnel = async () => {
+const collectNgrokStats = async (rerun = true) => {
+    try {
+        const ngrok = await lazyRequire('ngrok');
+        const api = ngrok.getApi();
+        const { tunnels } = await api.get({ url: 'api/tunnels', json: true });
+        const tunnel = tunnels.find(t => t.public_url === ngrokTunnelUrl);
+
+        // eslint-disable-next-line no-console
+        console.log('ngrok stats', tunnel);
+        logger.info('ngrok stats', { tunnel });
+    } catch (err) {
+        logger.error('error collecting ngrok stats', { err });
+    }
+    if (rerun) {
+        statsTimeout = setTimeout(() => collectNgrokStats(), 10000);
+    }
+};
+
+const disconnectTunnel = async (options) => {
     if (!ngrokTunnelUrl) {
         return;
     }
 
+    clearTimeout(statsTimeout);
+    if (options.tunnelDiagnostics) {
+        await collectNgrokStats(false);
+    }
     const ngrok = await lazyRequire('ngrok');
     await ngrok.disconnect(ngrokTunnelUrl);
 };

package/commons/testimTunnel.js

@@ -41,7 +41,7 @@ const disconnect = async (options) => {
         if (shouldUseLambdatestTunnel(options)) {
             await LambdatestService.disconnectTunnel(options);
         } else {
-            await testimNgrok.disconnectTunnel();
+            await testimNgrok.disconnectTunnel(options);
         }
     } catch (err) {
         const msg = 'catch error - failed to close tunnel';