browsertime 16.13.1 → 16.14.0

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 16.14.0 - 2022-08-26
+### Added
+* Collect relative standard deviation and output that in the cli output [#1834](https://github.com/sitespeedio/browsertime/pull/1834). Before we only showed the actual standard deviation, with this fix we also show it in percentage, that makes it easier if you run tests on different hardware/networks and want to compare them.
+* Collect number of processes that runs on the server that do the testing, before a test starts [#1833](https://github.com/sitespeedio/browsertime/pull/1833).
+## 16.13.3 - 2022-08-17
+
+### Fixed
+* Make sure set cookies work when navigating in script for Chrome/Edge [#1830](https://github.com/sitespeedio/browsertime/pull/1830).
+
+## 16.13.2 - 2022-08-14
+### Fixed
+* Disable adding lastCPULongTask to the HAR data since it breaks waterfall view on slow mobile phones with late CPU long tasks [#1828](https://github.com/sitespeedio/browsertime/pull/1828).
+
 ## 16.13.1 - 2022-08-10
 ### Fixed
 * Make sure that Android id is always picked up from the phone if tests run on Android [#1826](https://github.com/sitespeedio/browsertime/pull/1826).

package/lib/chrome/webdriver/chromium.js

@@ -444,6 +444,10 @@ class Chromium {
       return { har: harBuilder.mergeHars(this.hars) };
     } else return {};
   }
+
+  async setCookies(url, cookies) {
+    return this.cdpClient.setCookies(url, cookies);
+  }
 }
 
 module.exports = Chromium;

package/lib/core/engine/collector.js

@@ -48,7 +48,8 @@ function getNewResult(url, options) {
     extras: [],
     fullyLoaded: [],
     mainDocumentTimings: [],
-    errors: []
+    errors: [],
+    server: { processesAtStart: [] }
   };
 }
 
@@ -183,9 +184,6 @@ class Collector {
 
       if (allData.batteryTemperature) {
         results.android.batteryTemperature.push(allData.batteryTemperature);
-        statistics.addDeep({
-          android: { batteryTemperature: allData.batteryTemperature }
-        });
       }
 
       if (allData.markedAsFailure) {
@@ -193,6 +191,9 @@ class Collector {
         results.failureMessages = allData.failureMessages;
       }
 
+      if (allData.processesAtStart) {
+        results.server.processesAtStart.push(allData.processesAtStart);
+      }
       // If we don't have an error, use an empty array. In the future we want to push
       // more than one errors per run (maybe).
       results.errors.push(data.error || []);

package/lib/core/engine/command/measure.js

@@ -141,6 +141,15 @@ class Measure {
     log.info('Navigating to url %s iteration %s', url, this.index);
     if (this.numberOfVisitedPages === 0) {
       await this.extensionServer.setup(url, this.browser, this.options);
+
+      // There's a bug that we introduced when moving cookie handling to CDP
+      // and this is the hack to fix that.
+      if (
+        (this.options.cookie && this.options.browser === 'chrome') ||
+        this.options.browser === 'edge'
+      ) {
+        await this.engineDelegate.setCookies(url, this.options.cookie);
+      }
     }
     if (this.numberOfVisitedPages === 0) {
       await this.engineDelegate.beforeStartIteration(this.browser, this.index);

package/lib/core/engine/index.js

@@ -309,7 +309,6 @@ class Engine {
     }
 
     const totalResult = collector.getResults();
-
     // Add extra fields to the HAR
     // to make the HAR files better when we use them in
     // compare.sitespeed.io

package/lib/core/engine/iteration.js

@@ -1,5 +1,5 @@
 'use strict';
-
+const os = require('os');
 const webdriver = require('selenium-webdriver');
 const log = require('intel').getLogger('browsertime');
 const SeleniumRunner = require('../seleniumRunner');
@@ -35,6 +35,7 @@ const {
   ContextClick,
   MouseMove
 } = require('./command/mouse/');
+const getNumberOfRunningProcesses = require('../../support/processes');
 
 const { isAndroidConfigured, Android } = require('../../android');
 const Scroll = require('./command/scroll');
@@ -84,6 +85,9 @@ class Iteration {
    * @param {*} index - Which iteration it is
    */
   async run(navigationScript, index) {
+    if (os.platform() === 'darwin' || os.platform() === 'linux') {
+      this.processesAtStart = new Number(await getNumberOfRunningProcesses());
+    }
     const screenshotManager = new ScreenshotManager(
       this.storageManager,
       this.options
@@ -316,6 +320,8 @@ class Iteration {
       if (commands.meta.title) {
         result.title = commands.meta.title;
       }
+
+      result.processesAtStart = this.processesAtStart;
       return result;
     } catch (e) {
       log.error(e);

package/lib/support/har/index.js

@@ -91,9 +91,11 @@ function addExtrasToHAR(
     harPageTimings._firstPaint = timings.firstPaint;
   }
 
+  /*
   if (cpu && cpu.longTasks && cpu.longTasks.lastLongTask) {
     harPageTimings._lastCPULongTask = cpu.longTasks.lastLongTask;
   }
+  */
 
   if (timings && timings.largestContentfulPaint) {
     harPageTimings._largestContentfulPaint =

package/lib/support/processes.js

@@ -0,0 +1,7 @@
+'use strict';
+
+const execa = require('execa');
+module.exports = async function () {
+  const { stdout } = await execa.command('ps aux | wc -l', { shell: true });
+  return stdout;
+};

package/lib/support/statistics.js

@@ -168,6 +168,7 @@ class Statistics {
         median: parseFloat(stats.median().toFixed(decimals)),
         mean: parseFloat(stats.amean().toFixed(decimals)),
         mdev: parseFloat((stats.stddev() / Math.sqrt(stats.length)).toFixed(4)), // "standard deviation of the mean"
+        rsd: parseFloat((100 * stats.stddev()) / stats.amean()), // Relative standard deviation
         stddev: parseFloat(stats.stddev().toFixed(decimals))
       };
       percentiles.forEach(p => {

package/lib/support/util.js

@@ -18,7 +18,9 @@ module.exports = {
 
     let formatted = `${name}: ${fmt(multiple ? metric.median : metric)}`;
     if (extras) {
-      formatted += ` (σ${fmt(metric.stddev.toFixed(2))})`;
+      formatted += ` (σ${fmt(metric.stddev.toFixed(2))} ${
+        metric.stddev > 0 ? metric.rsd.toFixed(1) : '0'
+      }%)`;
     }
     return formatted;
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "description": "Browsertime",
-  "version": "16.13.1",
+  "version": "16.14.0",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@cypress/xvfb": "1.2.4",
@@ -8,7 +8,7 @@
     "@sitespeed.io/chromedriver": "104.0.5112-29",
     "@sitespeed.io/edgedriver": "104.0.1293-47",
     "@sitespeed.io/geckodriver": "0.31.0-c",
-    "@sitespeed.io/throttle": "4.0.1",
+    "@sitespeed.io/throttle": "5.0.0",
     "@sitespeed.io/tracium": "0.3.3",
     "btoa": "1.2.1",
     "chrome-har": "0.13.0",
@@ -26,7 +26,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.3.1",
+    "selenium-webdriver": "4.4.0",
     "yargs": "17.4.1"
   },
   "optionalDependencies": {