browsertime 14.20.2 → 15.0.0

package/CHANGELOG.md

@@ -1,6 +1,26 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
-## 14.20.2 - 2022-02-07
+## 15.0.0 - 2022-02-20
+### Changed
+* Removed the built in support for TSProxy [#1718](https://github.com/sitespeedio/browsertime/pull/1718). TSProxy only worked with Python 2.7 and we want to remove that dependency.
+* Removed support for using speedline and the CLI arg `--chrome.visualMetricsUsingTrace` [#1727](https://github.com/sitespeedio/browsertime/pull/1727).
+
+### Fixed
+* Code cleanup: [#1720](https://github.com/sitespeedio/browsertime/pull/1720)
+* Moved test to [Ava](https://avajs.dev) instead of mocha and moved most testing against a local HTTP server instead of internet [#1725](https://github.com/sitespeedio/browsertime/pull/1725).
+
+### Added
+* New alias for getting the Chrome trace log: `--chrome.trace` [#1728](https://github.com/sitespeedio/browsertime/pull/1728).
+* On OS X and Linux the DNS will be flushed between runs. You can disable that with `--disableDNSFlush true` [#1732](https://github.com/sitespeedio/browsertime/pull/1732).
+## 14.21.1 - 2022-02-09
+### Fixed
+* Always disable the new ChromeWhaxyNewUI splash screen in Chrome [#1717](https://github.com/sitespeedio/browsertime/pull/1717).
+* Remove the Firefox persist log configuration that was needed for FF 94+ [#1716](https://github.com/sitespeedio/browsertime/pull/1716).
+## 14.21.0 - 2022-02-08
+### Added
+* Upgraded to Egdedriver 98
+* Upgraded to Edge 98 and Firefox 97 in the Docker container
+## 14.20.3 - 2022-02-07
 ### Fixed
 * There's a new setup how we build Docker releases and this release uses the new pipeline.
 ## 14.20.0 - 2022-02-07

package/README.md

@@ -1,9 +1,9 @@
 # Browsertime - Your browser, your page, your scripts!
-![Run Docker](https://github.com/sitespeedio/browsertime/workflows/Run%20Docker/badge.svg?branch=main)
-![Unit tests](https://github.com/sitespeedio/browsertime/workflows/Unit%20tests/badge.svg?branch=main)
-![Windows Edge](https://github.com/sitespeedio/browsertime/workflows/Windows%20Edge/badge.svg?branch=main)
-![OSX Safari](https://github.com/sitespeedio/browsertime/workflows/OSX%20Safari/badge.svg?branch=main)
-![Linux browsers](https://github.com/sitespeedio/browsertime/workflows/Linux%20browsers/badge.svg?branch=main)
+![Run Docker](https://github.com/sitespeedio/browsertime/actions/workflows/docker.yml/badge.svg?branch=main)
+![Unit tests](https://github.com/sitespeedio/browsertime/actions/workflows/unittests.yml/badge.svg?branch=main)
+![Windows Edge](https://github.com/sitespeedio/browsertime/actions/workflows/windows.yml/badge.svg?branch=main)
+![OSX Safari](https://github.com/sitespeedio/browsertime/actions/workflows/safari.yml/badge.svg?branch=main)
+![Linux browsers](https://github.com/sitespeedio/browsertime/actions/workflows/linux.yml/badge.svg?branch=main)
 [![Downloads][downloads-image]][downloads-url]
 [![Downloads total][downloads-total-image]][downloads-url]
 [![Stars][stars-image]][stars-url]

package/lib/chrome/webdriver/chromium.js

@@ -7,7 +7,6 @@ const cpuMetrics = require('../longTaskMetrics');
 const fs = require('fs');
 const path = require('path');
 const parseCpuTrace = require('../parseCpuTrace');
-const speedline = require('../speedline');
 const har = require('../har');
 const harBuilder = require('../../support/har');
 const webdriver = require('selenium-webdriver');
@@ -286,14 +285,6 @@ class Chromium {
       const cpu = await parseCpuTrace(trace, result.url);
       result.cpu = cpu;
 
-      if (
-        this.chrome.enableTraceScreenshots &&
-        this.chrome.visualMetricsUsingTrace &&
-        this.options.cpu
-      ) {
-        result.visualMetrics = speedline(trace, result, index);
-      }
-
       // Collect render blocking info
       const renderBlockingInfo = {};
       const urlsWithBlockingInfo = trace.traceEvents.filter(

package/lib/chrome/webdriver/setupChromiumOptions.js

@@ -108,9 +108,25 @@ module.exports = function (seleniumOptions, browserOptions, options, baseDir) {
     }
   }
 
+  // It's a new splash screen introduced in Chrome 98
+  // for new profiles
+  // disable it with ChromeWhatsNewUI
   if (browserOptions.args) {
-    log.debug('Set Chrome args %j', browserOptions.args);
-    seleniumOptions.addArguments(browserOptions.args);
+    const chromeCommandLineArgs = util.toArray(browserOptions.args);
+    for (const arg of chromeCommandLineArgs) {
+      if (
+        arg.indexOf('disable-features') > -1 &&
+        arg.indexOf('ChromeWhatsNewUI') === -1
+      ) {
+        seleniumOptions.addArguments(`${arg},ChromeWhatsNewUI`);
+        log.debug('Set Chrome args %j', `${arg},ChromeWhatsNewUI`);
+      } else {
+        seleniumOptions.addArguments(arg);
+        log.debug('Set Chrome args %j', arg);
+      }
+    }
+  } else {
+    seleniumOptions.addArguments('--disable-features=ChromeWhatsNewUI');
   }
 
   if (browserOptions.binaryPath) {
@@ -121,33 +137,6 @@ module.exports = function (seleniumOptions, browserOptions, options, baseDir) {
     seleniumOptions.setMobileEmulation(browserOptions.mobileEmulation);
   }
 
-  const connectivity = options.connectivity || {};
-
-  if (
-    connectivity.engine === 'tsproxy' &&
-    connectivity.tsproxy &&
-    connectivity.profile !== 'native'
-  ) {
-    if (!connectivity.tsproxy.bind) {
-      // If you run tsproxy on desktop instead of throttle or Docker networs
-      // Configure SOCKS proxy, see https://www.chromium.org/developers/design-documents/network-stack/socks-proxy
-      seleniumOptions.addArguments(
-        '--proxy-server=socks5://localhost:' + connectivity.tsproxy.port
-      );
-      seleniumOptions.addArguments(
-        '--host-resolver-rules="MAP * ~NOTFOUND , EXCLUDE localhost"'
-      );
-    } else {
-      // If you run on Android and want to use TSProxy on your desktop
-      seleniumOptions.addArguments(
-        '--proxy-server=socks://' +
-          connectivity.tsproxy.bind +
-          ':' +
-          connectivity.tsproxy.port
-      );
-    }
-  }
-
   // See https://bugs.chromium.org/p/chromium/issues/detail?id=818483
   // Coming again in Chrome 76
   seleniumOptions.excludeSwitches('enable-automation');

package/lib/connectivity/index.js

@@ -4,10 +4,8 @@ const trafficShapeParser = require('./trafficShapeParser');
 const get = require('lodash.get');
 const throttle = require('@sitespeed.io/throttle');
 const log = require('intel').getLogger('browsertime.connectivity');
-const TSProxy = require('./tsProxy');
 const Humble = require('./humble');
 
-let tsProxyInstance = null;
 let humble = null;
 
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random
@@ -71,10 +69,6 @@ module.exports = {
           });
         }
       }
-      case 'tsproxy': {
-        tsProxyInstance = new TSProxy(options);
-        return tsProxyInstance.start(profile);
-      }
       case 'humble': {
         humble = new Humble(options);
         return humble.start(profile);
@@ -99,9 +93,6 @@ module.exports = {
         const setOnLocalHost = get(connectivity, 'throttle.localhost', false);
         return throttle.stop({ localhost: setOnLocalHost });
       }
-      case 'tsproxy': {
-        return tsProxyInstance.stop();
-      }
       case 'humble': {
         return humble.stop();
       }

package/lib/core/engine/index.js

@@ -6,7 +6,6 @@ const getPort = require('get-port');
 const execa = require('execa');
 const os = require('os');
 const get = require('lodash.get');
-const set = require('lodash.set');
 const StorageManager = require('../../support/storageManager');
 const Firefox = require('../../firefox/webdriver/firefox');
 const Chromium = require('../../chrome/webdriver/chromium');
@@ -58,46 +57,32 @@ class Engine {
    * * Start the extension server
    */
   async start() {
-    this.options.devToolsPort = await getPort({
+    const options = this.options;
+
+    options.devToolsPort = await getPort({
       port: getPort.makeRange(9222, 9350),
       host: '127.0.0.1'
     });
 
-    this.options.safariDriverPort = await getPort({
+    options.safariDriverPort = await getPort({
       port: getPort.makeRange(1234, 2000),
       host: '127.0.0.1'
     });
 
-    if (
-      this.options.browser === 'safari' &&
-      this.options.safari.ios &&
-      this.options.video
-    ) {
+    if (options.browser === 'safari' && options.safari.ios && options.video) {
       await IOSRecorder.activate();
       await delay(2000);
     }
 
-    if (get(this.options, 'connectivity.tsproxy.port') === undefined) {
-      set(
-        this.options,
-        'connectivity.tsproxy.port',
-        await getPort({
-          port: getPort.makeRange(1080, 1099),
-          host: '127.0.0.1'
-        })
-      );
-    }
-    if (!this.options.connectivity.variance) {
-      await addConnectivity(this.options);
-    }
+    await addConnectivity(options);
 
-    if (this.options.android) {
-      const android = new Android(this.options);
+    if (options.android) {
+      const android = new Android(options);
       await android._init();
       this.android = android;
 
-      if (this.options.gnirehtet) {
-        this.gnirehtet = new Gnirehtet(this.options);
+      if (options.gnirehtet) {
+        this.gnirehtet = new Gnirehtet(options);
         await this.gnirehtet.start();
       }
 
@@ -108,12 +93,12 @@ class Engine {
         throw Error(message);
       }
 
-      if (this.options.androidPretestPowerPress) {
+      if (options.androidPretestPowerPress) {
         await android.clickPowerButton();
       }
 
-      if (this.options.androidVerifyNetwork) {
-        const pingAddress = get(this.options, 'androidPingAddress', '8.8.8.8');
+      if (options.androidVerifyNetwork) {
+        const pingAddress = get(options, 'androidPingAddress', '8.8.8.8');
         const connection = await android.ping(pingAddress);
         if (!connection) {
           const message = `No internet connection for ${android.id}. Could not ping ${pingAddress}`;
@@ -127,7 +112,7 @@ class Engine {
       // Make sure screen is on!
       await android._runCommand('svc power stayon true');
 
-      if (this.options.androidRooted) {
+      if (options.androidRooted) {
         const gotRoot = await android._runCommandAndGet(
           'su - root -c "echo test"'
         );
@@ -139,14 +124,13 @@ class Engine {
         }
       }
 
-      if (this.options.androidBatteryTemperatureLimit) {
-        const maxTries = this.options.androidBatteryTemperatureMaxTries || 20;
+      if (options.androidBatteryTemperatureLimit) {
+        const maxTries = options.androidBatteryTemperatureMaxTries || 20;
         let batteryTry = 0;
         let temp = await android.getTemperature();
-        const minTempLimit = this.options.androidBatteryTemperatureLimit;
+        const minTempLimit = options.androidBatteryTemperatureLimit;
         const waitTime =
-          (this.options.androidBatteryTemperatureWaitTimeInSeconds || 120) *
-          1000;
+          (options.androidBatteryTemperatureWaitTimeInSeconds || 120) * 1000;
         if (temp > minTempLimit) {
           do {
             temp = await android.getTemperature();
@@ -164,7 +148,7 @@ class Engine {
                 minTempLimit,
                 maxTries
               );
-              if (this.options.androidBatteryTemperatureReboot) {
+              if (options.androidBatteryTemperatureReboot) {
                 log.info(
                   'Rebooting the device, will wait until ADB sees the device'
                 );
@@ -181,16 +165,13 @@ class Engine {
         }
       }
 
-      if (
-        this.options.connectivity &&
-        this.options.connectivity.engine === 'humble'
-      ) {
+      if (options.connectivity && options.connectivity.engine === 'humble') {
         const wifiName = await android.getWifi();
         log.info('The phone is using the WiFi: %s', wifiName);
       }
     }
 
-    if (this.options.safari && this.options.safari.useSimulator) {
+    if (options.safari && options.safari.useSimulator) {
       // Start the simulator
       try {
         await execa.command(
@@ -211,7 +192,7 @@ class Engine {
       const types = Object.keys(simulatedDevices.devices);
       for (let type of types) {
         for (let device of simulatedDevices.devices[type]) {
-          if (device.udid === this.options.safari.deviceUDID) {
+          if (device.udid === options.safari.deviceUDID) {
             this.iosSimulatorDeviceName = device.name;
             log.info(
               'Running test on %s simulator using %s',
@@ -241,16 +222,16 @@ class Engine {
     let engineDelegate;
     switch (options.browser) {
       case 'firefox':
-        engineDelegate = new Firefox(storageManager, this.options);
+        engineDelegate = new Firefox(storageManager, options);
         break;
       case 'chrome':
-        engineDelegate = new Chromium(storageManager, this.options);
+        engineDelegate = new Chromium(storageManager, options);
         break;
       case 'edge':
-        engineDelegate = new Chromium(storageManager, this.options);
+        engineDelegate = new Chromium(storageManager, options);
         break;
       case 'safari':
-        engineDelegate = new Safari(storageManager, this.options);
+        engineDelegate = new Safari(storageManager, options);
     }
     const iteration = new Iteration(
       storageManager,
@@ -388,11 +369,11 @@ class Engine {
       }
     }
 
-    if (this.options.safari && this.options.safari.useSimulator) {
+    if (options.safari && options.safari.useSimulator) {
       for (let result of totalResult) {
         result.info.ios = {
           deviceName: this.iosSimulatorDeviceName + ' simulator',
-          deviceUDID: this.options.safari.deviceUDID,
+          deviceUDID: options.safari.deviceUDID,
           arch: os.arch()
         };
       }
@@ -410,6 +391,7 @@ class Engine {
   }
 
   async runMultiple(urlOrFiles, scriptsByCategory, asyncScriptsByCategory) {
+    const options = this.options;
     const scripts = [];
     let name;
     for (let urlOrFile of urlOrFiles) {
@@ -423,17 +405,17 @@ class Engine {
 
         script = engineUtils.loadScript(script, true);
         if (script.setUp) {
-          if (!this.options.preScript) {
-            this.options.preScript = [];
+          if (!options.preScript) {
+            options.preScript = [];
           }
-          this.options.preScript.push(script.setUp);
+          options.preScript.push(script.setUp);
         }
 
         if (script.tearDown) {
-          if (!this.options.postScript) {
-            this.options.postScript = [];
+          if (!options.postScript) {
+            options.postScript = [];
           }
-          this.options.postScript.push(script.tearDown);
+          options.postScript.push(script.tearDown);
         }
         // here, url is the filename containing the script, and test the callable.
         if (script.test) {
@@ -461,15 +443,14 @@ class Engine {
    * * Stop the extension server
    */
   async stop() {
-    if (!this.options.connectivity.variance) {
-      await removeConnectivity(this.options);
-    }
+    const options = this.options;
+    await removeConnectivity(options);
 
-    if (this.options.androidRooted && this.rooted) {
+    if (options.androidRooted && this.rooted) {
       await this.rooted.stop();
     }
 
-    if (this.options.safari && this.options.safari.useSimulator) {
+    if (options.safari && options.safari.useSimulator) {
       try {
         await execa.command('pkill -x Simulator', { shell: true });
       } catch (e) {
@@ -477,7 +458,7 @@ class Engine {
       }
     }
 
-    if (this.options.gnirehtet && this.gnirehtet) {
+    if (options.gnirehtet && this.gnirehtet) {
       await this.gnirehtet.stop();
     }
 

package/lib/core/engine/iteration.js

@@ -26,6 +26,7 @@ const AndroidCommand = require('./command/android.js');
 const ChromeDevToolsProtocol = require('./command/chromeDevToolsProtocol.js');
 const { addConnectivity, removeConnectivity } = require('../../connectivity');
 const util = require('../../support/util.js');
+const flushDNS = require('../../support/dns.js');
 const {
   SingleClick,
   DoubleClick,
@@ -101,9 +102,11 @@ class Iteration {
     const engineDelegate = this.engineDelegate;
 
     try {
-      if (options.connectivity.variance) {
-        await addConnectivity(options);
+      if (!options.disableDNSFlush === true) {
+        await flushDNS(options);
       }
+      await addConnectivity(options);
+
       await engineDelegate.beforeBrowserStart();
       await browser.start();
       await engineDelegate.afterBrowserStart();

package/lib/core/seleniumRunner.js

@@ -76,21 +76,11 @@ class SeleniumRunner {
         );
         break;
       } catch (e) {
-        // Fail fast if you are using the incorrect Chromedriver version
-        if (
-          e.message.indexOf('This version of ChromeDriver only supports') > -1
-        ) {
-          log.error(
-            'There is a miss match between Chrome and Chromedriver version. You can manually install the correct Chromedriver version, download the correct version from https://chromedriver.chromium.org/downloads and set the path to the file with --chrome.chromedriverPath see https://www.sitespeed.io/documentation/sitespeed.io/browsers/#use-a-newer-version-of-chromedriver'
-          );
-          throw e;
-        } else {
-          log.info(
-            `${this.options.browser} failed to start, trying ${
-              tries - i - 1
-            } more time(s): ${e.message}`
-          );
-        }
+        log.info(
+          `${this.options.browser} failed to start, trying ${
+            tries - i - 1
+          } more time(s): ${e.message}`
+        );
       }
     }
     if (!this.driver) {

package/lib/firefox/settings/firefoxPreferences.js

@@ -179,8 +179,5 @@ module.exports = {
 
   // disable calls to detectportal.firefox.com
   'network.captive-portal-service.enabled': false,
-  'network.connectivity-service.enabled': false,
-  // https://bugzilla.mozilla.org/show_bug.cgi?id=1740116#c15
-  // https://github.com/sitespeedio/browsertime/issues/1671
-  'devtools.netmonitor.persistlog': true
+  'network.connectivity-service.enabled': false
 };

package/lib/firefox/webdriver/builder.js

@@ -207,20 +207,6 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
     builder.getCapabilities().set('acceptInsecureCerts', true);
   }
 
-  const connectivity = options.connectivity || {};
-
-  if (
-    connectivity.engine === 'tsproxy' &&
-    connectivity.tsproxy &&
-    connectivity.profile !== 'native'
-  ) {
-    ffOptions.setPreference('network.proxy.socks', 'localhost');
-    ffOptions.setPreference(
-      'network.proxy.socks_port',
-      connectivity.tsproxy.port
-    );
-    ffOptions.setPreference('network.proxy.type', 1);
-  }
   builder
     .getCapabilities()
     .set('pageLoadStrategy', get(options, 'pageLoadStrategy', 'normal'));

package/lib/firefox/webdriver/firefox.js

@@ -143,6 +143,24 @@ class Firefox {
       result.memory = await memoryReport.collect(index, url);
     }
 
+    const useFirefoxAppConstants = get(
+      this.options || {},
+      'firefox.appconstants',
+      false
+    );
+
+    if (useFirefoxAppConstants === true) {
+      const appConstantsScript = `const { AppConstants } = ChromeUtils.import(
+      'resource://gre/modules/AppConstants.jsm'
+    );
+    return AppConstants;`;
+
+      this.appConstants = await runner.runPrivilegedScript(
+        appConstantsScript,
+        'APP_CONSTANTS'
+      );
+    }
+
     if (this.skipHar || this.options.android) {
       return result;
     } else {
@@ -167,27 +185,8 @@ class Firefox {
    * The URL/test is finished, all metrics are collected.
    */
   async afterEachURL(runner, index, result) {
-    const useFirefoxAppConstants = get(
-      this.options || {},
-      'firefox.appconstants',
-      false
-    );
-
-    if (useFirefoxAppConstants === false) {
-      // We need to be extra careful here, see
-      // https://github.com/sitespeedio/browsertime/issues/1261
-      if (
-        result.browserScripts.browser &&
-        result.browserScripts.browser.appConstants
-      ) {
-        delete result.browserScripts.browser.appConstants;
-      }
-      if (
-        result.browserScripts.browser &&
-        result.browserScripts.browser.asyncAppConstants
-      ) {
-        delete result.browserScripts.browser.asyncAppConstants;
-      }
+    if (this.appConstants) {
+      result.browserScripts.browser.appConstants = this.appConstants;
     }
   }
 

package/lib/support/cli.js

@@ -87,12 +87,6 @@ function validateInput(argv) {
     }
   }
 
-  if (argv.connectivity && argv.connectivity.engine === 'tsproxy') {
-    if (!hasbin.all.sync(['python'])) {
-      return 'You need to have Python 2.7 in your path to be able to use TSProxy.';
-    }
-  }
-
   if (argv.connectivity && argv.connectivity.engine === 'humble') {
     if (!argv.connectivity.humble || !argv.connectivity.humble.url) {
       return 'You need to specify the URL to Humble by using the --connectivity.humble.url option.';
@@ -253,14 +247,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
       type: 'boolean',
       group: 'chrome'
     })
-    .option('chrome.visualMetricsUsingTrace', {
-      type: 'boolean',
-      default: false,
-      describe:
-        'Collect Visual Metrics using Chrome trace log. You need enable trace screenshots --chrome.enableTraceScreenshots and --cpu metrics for this to work.',
-      group: 'chrome'
-    })
     .option('chrome.timeline', {
+      alias: 'chrome.trace',
       describe:
         'Collect the timeline data. Drag and drop the JSON in your Chrome detvools timeline panel or check out the CPU metrics in the Browsertime.json',
       type: 'boolean',
@@ -943,7 +931,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
     })
     .option('connectivity.engine', {
       default: 'external',
-      choices: ['external', 'throttle', 'tsproxy', 'humble'],
+      choices: ['external', 'throttle', 'humble'],
       describe:
         'The engine for connectivity. Throttle works on Mac and tc based Linux. For mobile you can use Humble if you have a Humble setup. Use external if you set the connectivity outside of Browsertime. The best way do to this is described in https://github.com/sitespeedio/browsertime#connectivity.',
       group: 'connectivity'

package/lib/support/dns.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const execa = require('execa');
+const log = require('intel').getLogger('browsertime.dns');
+const { isAndroidConfigured } = require('../android');
+module.exports = async function (options) {
+  if (isAndroidConfigured(options)) {
+    return;
+  }
+
+  if (options.safari && options.safari.ios) {
+    return;
+  }
+
+  if (process.platform === 'darwin') {
+    try {
+      await execa('sudo', ['killall', '-HUP', 'mDNSResponder'], {
+        reject: false
+      });
+      await execa('sudo', ['dscacheutil', '-flushcache'], {
+        reject: false
+      });
+      return execa('sudo', ['lookupd', '-flushcache'], {
+        reject: false
+      });
+    } catch (err) {
+      log.error('Could not flush dns cache', err);
+    }
+  } else if (process.platform === 'linux') {
+    try {
+      await execa('sudo', ['service', 'dnsmasq', 'restart'], {
+        reject: false
+      });
+      await execa('sudo', ['rndc', 'restart'], {
+        reject: false
+      });
+      return execa('sudo', ['systemd-resolve', '--flush-caches'], {
+        reject: false
+      });
+    } catch (err) {
+      log.error('Could not flush dns cache', err);
+    }
+  }
+};