browsertime 16.13.3 → 16.14.2

package/CHANGELOG.md

@@ -1,6 +1,19 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
 
+## 16.14.2 - 2022-08-26
+### Fixed
+* Using `--preWarmServer` didn't work on Android/iOS [#1837](https://github.com/sitespeedio/browsertime/pull/1837).
+## 16.14.1 - 2022-08-26
+### Fixed
+* Guard relative standard deviation agains 0 standard deviation [#1835](https://github.com/sitespeedio/browsertime/pull/1835).
+## 16.14.0 - 2022-08-26
+### Added
+* Collect relative standard deviation and output that in the cli output [#1834](https://github.com/sitespeedio/browsertime/pull/1834). Before we only showed the actual standard deviation, with this fix we also show it in percentage, that makes it easier if you run tests on different hardware/networks and want to compare them.
+* Collect number of processes that runs on the server that do the testing, before a test starts [#1833](https://github.com/sitespeedio/browsertime/pull/1833).
+
+### Fixed
+* Updated to Selenium 4.4.0 and Throttle 5.0.0.
 ## 16.13.3 - 2022-08-17
 
 ### Fixed

package/bin/browsertime.js

@@ -44,19 +44,22 @@ async function preWarmServer(urls, options) {
   const safariDeviceName = get(options, 'safari.deviceName');
   const safariDeviceUDID = get(options, 'safari.deviceUDID ');
 
+  if (options.android && options.browser === 'chrome') {
+    set(preWarmOptions, 'chrome.android.package', 'com.android.chrome');
+  }
   if (chromeDevice) {
-    set(options, 'chrome.android.deviceSerial', chromeDevice);
+    set(preWarmOptions, 'chrome.android.deviceSerial', chromeDevice);
   } else if (firefoxDevice) {
-    set(options, 'firefox.android.deviceSerial', firefoxDevice);
+    set(preWarmOptions, 'firefox.android.deviceSerial', firefoxDevice);
   }
 
   if (safariIos) {
-    set(options, 'safari.ios', true);
+    set(preWarmOptions, 'safari.ios', true);
     if (safariDeviceName) {
-      set(options, 'safari.deviceName', safariDeviceName);
+      set(preWarmOptions, 'safari.deviceName', safariDeviceName);
     }
     if (safariDeviceUDID) {
-      set(options, 'safari.deviceUDID', safariDeviceUDID);
+      set(preWarmOptions, 'safari.deviceUDID', safariDeviceUDID);
     }
   }
 

package/lib/core/engine/collector.js

@@ -48,7 +48,8 @@ function getNewResult(url, options) {
     extras: [],
     fullyLoaded: [],
     mainDocumentTimings: [],
-    errors: []
+    errors: [],
+    server: { processesAtStart: [] }
   };
 }
 
@@ -183,9 +184,6 @@ class Collector {
 
       if (allData.batteryTemperature) {
         results.android.batteryTemperature.push(allData.batteryTemperature);
-        statistics.addDeep({
-          android: { batteryTemperature: allData.batteryTemperature }
-        });
       }
 
       if (allData.markedAsFailure) {
@@ -193,6 +191,9 @@ class Collector {
         results.failureMessages = allData.failureMessages;
       }
 
+      if (allData.processesAtStart) {
+        results.server.processesAtStart.push(allData.processesAtStart);
+      }
       // If we don't have an error, use an empty array. In the future we want to push
       // more than one errors per run (maybe).
       results.errors.push(data.error || []);

package/lib/core/engine/index.js

@@ -309,7 +309,6 @@ class Engine {
     }
 
     const totalResult = collector.getResults();
-
     // Add extra fields to the HAR
     // to make the HAR files better when we use them in
     // compare.sitespeed.io

package/lib/core/engine/iteration.js

@@ -1,5 +1,5 @@
 'use strict';
-
+const os = require('os');
 const webdriver = require('selenium-webdriver');
 const log = require('intel').getLogger('browsertime');
 const SeleniumRunner = require('../seleniumRunner');
@@ -35,6 +35,7 @@ const {
   ContextClick,
   MouseMove
 } = require('./command/mouse/');
+const getNumberOfRunningProcesses = require('../../support/processes');
 
 const { isAndroidConfigured, Android } = require('../../android');
 const Scroll = require('./command/scroll');
@@ -84,6 +85,9 @@ class Iteration {
    * @param {*} index - Which iteration it is
    */
   async run(navigationScript, index) {
+    if (os.platform() === 'darwin' || os.platform() === 'linux') {
+      this.processesAtStart = new Number(await getNumberOfRunningProcesses());
+    }
     const screenshotManager = new ScreenshotManager(
       this.storageManager,
       this.options
@@ -316,6 +320,8 @@ class Iteration {
       if (commands.meta.title) {
         result.title = commands.meta.title;
       }
+
+      result.processesAtStart = this.processesAtStart;
       return result;
     } catch (e) {
       log.error(e);

package/lib/support/processes.js

@@ -0,0 +1,7 @@
+'use strict';
+
+const execa = require('execa');
+module.exports = async function () {
+  const { stdout } = await execa.command('ps aux | wc -l', { shell: true });
+  return stdout;
+};

package/lib/support/statistics.js

@@ -168,6 +168,10 @@ class Statistics {
         median: parseFloat(stats.median().toFixed(decimals)),
         mean: parseFloat(stats.amean().toFixed(decimals)),
         mdev: parseFloat((stats.stddev() / Math.sqrt(stats.length)).toFixed(4)), // "standard deviation of the mean"
+        rsd:
+          stats.stddev() > 0
+            ? parseFloat((100 * stats.stddev()) / stats.amean())
+            : 0, // Relative standard deviation
         stddev: parseFloat(stats.stddev().toFixed(decimals))
       };
       percentiles.forEach(p => {

package/lib/support/util.js

@@ -18,7 +18,9 @@ module.exports = {
 
     let formatted = `${name}: ${fmt(multiple ? metric.median : metric)}`;
     if (extras) {
-      formatted += ` (σ${fmt(metric.stddev.toFixed(2))})`;
+      formatted += ` (σ${fmt(metric.stddev.toFixed(2))} ${
+        metric.stddev > 0 ? metric.rsd.toFixed(1) : '0'
+      }%)`;
     }
     return formatted;
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "description": "Browsertime",
-  "version": "16.13.3",
+  "version": "16.14.2",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@cypress/xvfb": "1.2.4",
@@ -8,7 +8,7 @@
     "@sitespeed.io/chromedriver": "104.0.5112-29",
     "@sitespeed.io/edgedriver": "104.0.1293-47",
     "@sitespeed.io/geckodriver": "0.31.0-c",
-    "@sitespeed.io/throttle": "4.0.1",
+    "@sitespeed.io/throttle": "5.0.0",
     "@sitespeed.io/tracium": "0.3.3",
     "btoa": "1.2.1",
     "chrome-har": "0.13.0",
@@ -26,7 +26,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.3.1",
+    "selenium-webdriver": "4.4.0",
     "yargs": "17.4.1"
   },
   "optionalDependencies": {