browsertime 17.3.0 → 17.5.0-beta1

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 17.4.0 - 2022-03-29
+### Added
+* Log the CPU benchmark metric to the console. This is useful (at least for me) when debugging instances with a lot of instability. [#1920](https://github.com/sitespeedio/browsertime/pull/1920).
+* In Chrome 111 some metrics become more instable when running on host with limited CPU. This change also exists in 112 beta. However we where missing out on a couple of command line magic to disable features in Chrome, this fixes that [#1921](https://github.com/sitespeedio/browsertime/pull/1921).
+
 ## 17.3.0 - 2022-03-26
 
 ### Fixed
@@ -7,6 +12,8 @@
 
 ### Added
 * On Linux you can use taskset to assing FFMPEG to specific CPUs using `--videoParams.taskset "0,5,7,9-11" `. It will start FFMPEG with `taskset -c <CPUS>` to pin FFMPG to specific CPU(s). Specify a numerical list of processors. The list may contain multiple items, separated by comma, and ranges. For example, "0,5,7,9-11". Use it together with isolcpus. Added in [#1917](https://github.com/sitespeedio/browsertime/pull/1917).
+
+* If you use a rooted Samsung A51 you can now choose the CPU speed (min/middle/max) using `--android.pinCPUSpeed min|middle|max` [#1915](https://github.com/sitespeedio/browsertime/pull/1915).
 ## 17.2.1 - 2022-03-16
 ### Fixed
 * Bumped Geckodriver to 0.32.2 and Edgedriver to 111.

package/lib/chrome/settings/chromeDesktopOptions.js

@@ -1,26 +1,28 @@
-// See https://github.com/GoogleChrome/chrome-launcher/blob/master/docs/chrome-flags-for-tools.md
+// See https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
 // https://peter.sh/experiments/chromium-command-line-switches/
 export const chromeDesktopOptions = [
-  '--disable-background-networking',
-  '--no-default-browser-check',
-  '--no-first-run',
-  '--new-window',
   '--allow-running-insecure-content',
+  '--disable-background-networking',
+  '--disable-background-timer-throttling',
+  '--disable-backgrounding-occluded-windows',
+  '--disable-breakpad',
   '--disable-client-side-phishing-detection',
   '--disable-component-update',
   '--disable-default-apps',
-  '--disable-device-discovery-notifications',
+  '--disable-dev-shm-usage',
   '--disable-domain-reliability',
-  '--disable-background-timer-throttling',
-  '--load-media-router-component-extension=0',
-  '--mute-audio',
+  '--disable-fetching-hints-at-navigation-start',
   '--disable-hang-monitor',
+  '--disable-ipc-flooding-protection',
+  '--disable-prompt-on-repost',
+  '--disable-renderer-backgrounding',
+  '--disable-site-isolation-trials',
+  '--disable-sync',
+  '--metrics-recording-only',
+  '--mute-audio',
+  '--new-window',
+  '--no-default-browser-check',
+  '--no-first-run',
   '--password-store=basic',
-  '--disable-breakpad',
-  '--dont-require-litepage-redirect-infobar',
-  '--override-https-image-compression-infobar',
-  '--disable-fetching-hints-at-navigation-start',
-  '--disable-dev-shm-usage',
-  '--disable-back-forward-cache',
-  '--disable-site-isolation-trials'
+  '--use-mock-keychain'
 ];

package/lib/chrome/webdriver/setupChromiumOptions.js

@@ -19,6 +19,17 @@ const CHROME_AMD_EDGE_INTERNAL_PHONE_HOME = [
   '"MAP optimizationguide-pa.googleapis.com 127.0.0.1"'
 ];
 
+const CHROME_FEATURES_THAT_WE_DISABLES = [
+  'AutofillServerCommunication',
+  'CalculateNativeWinOcclusion',
+  'HeavyAdPrivacyMitigations',
+  'InterestFeedContentSuggestions',
+  'MediaRouter',
+  'OfflinePagesPrefetching',
+  'OptimizationHints',
+  'Translate'
+];
+
 export function setupChromiumOptions(
   seleniumOptions,
   browserOptions,
@@ -49,6 +60,10 @@ export function setupChromiumOptions(
     seleniumOptions.addArguments('--disable-gpu');
   }
 
+  seleniumOptions.addArguments(
+    '--disable-features=' + CHROME_FEATURES_THAT_WE_DISABLES.join(',')
+  );
+
   const viewPort = getViewPort(options);
   // If viewport is defined (only on desktop) then set start args
   if (viewPort) {

package/lib/core/engine/iteration.js

@@ -112,7 +112,7 @@ export class Iteration {
 
       await engineDelegate.beforeBrowserStart();
       await browser.start();
-      await engineDelegate.afterBrowserStart();
+      await engineDelegate.afterBrowserStart(browser);
 
       // The data we push to all selenium scripts
       const context = {

package/lib/firefox/webdriver/builder.js

@@ -116,24 +116,6 @@ export async function configureBuilder(builder, baseDir, options) {
     ffOptions.setPreference('detach', true);
   }
 
-  if (!options.skipHar) {
-    if (isAndroidConfigured(options)) {
-      log.info('Skip install HAR trigger on Android');
-    } else {
-      // Hack for opening the toolbar
-      // In Firefox 61 we need to have devtools open but do not need to choose netmonitor
-      // ffOptions.setPreference('devtools.toolbox.selectedTool', 'netmonitor');
-      if (!options.debug) {
-        ffOptions.setPreference('devtools.toolbox.footer.height', 0);
-        ffOptions.addArguments('-devtools');
-      }
-
-      ffOptions.addExtensions(
-        resolve(moduleRootPath, 'vendor', 'har_export_trigger-0.6.1-an+fx.xpi')
-      );
-    }
-  }
-
   // Browsertime own extension, only load it when we need it
   // We should be able to only install it only when needed, but that could break scripting
 
@@ -193,6 +175,9 @@ export async function configureBuilder(builder, baseDir, options) {
 
   ffOptions.addArguments('-no-remote');
 
+  // Enable bidi for HAR support
+  ffOptions.enableBidi();
+
   if (firefoxConfig.args) {
     ffOptions.addArguments(firefoxConfig.args);
   }

package/lib/firefox/webdriver/firefox.js

@@ -3,15 +3,16 @@ import { promisify } from 'node:util';
 import { join } from 'node:path';
 import intel from 'intel';
 import get from 'lodash.get';
+import { adapters } from 'ff-test-bidi-har-export';
 import { getEmptyHAR, mergeHars } from '../../support/har/index.js';
 import { pathToFolder } from '../../support/pathToFolder.js';
 import { isAndroidConfigured, Android } from '../../android/index.js';
 import { adjustVisualProgressTimestamps } from '../../support/util.js';
 import { findFiles } from '../../support/fileUtil.js';
-import { getHAR } from '../getHAR.js';
 import { GeckoProfiler } from '../geckoProfiler.js';
 import { MemoryReport } from '../memoryReport.js';
 import { PerfStats } from '../perfStats.js';
+
 const log = intel.getLogger('browsertime.firefox');
 const rename = promisify(_rename);
 export class Firefox {
@@ -48,7 +49,13 @@ export class Firefox {
    * The browser is up and running, now its time to start to
    * configure what you need.
    */
-  async afterBrowserStart() {}
+  async afterBrowserStart(runner) {
+    const windowId = await runner.getDriver().getWindowHandle();
+    this.har = new adapters.SeleniumBiDiHarRecorder({
+      browsingContextIds: [windowId],
+      driver: runner.getDriver()
+    });
+  }
 
   /**
    * Before the first iteration of your tests start.
@@ -81,6 +88,10 @@ export class Firefox {
       });
     `);
 
+    if (!this.skipHar) {
+      await this.har.startRecording();
+    }
+
     if (isAndroidConfigured(this.options) && this.options.androidPower) {
       await this.android.resetPowerUsage();
     }
@@ -166,10 +177,11 @@ export class Firefox {
     }
     result.cpu = powerusage;
 
-    if (this.skipHar || isAndroidConfigured(this.options)) {
+    if (this.skipHar) {
       return result;
     } else {
-      const har = await getHAR(runner, this.includeResponseBodies);
+      // const harExport = await this.har.stopRecording();
+      const har = await this.har.stopRecording();
       if (har.log.pages.length > 0) {
         // Hack to add the URL from a SPA
         if (result.alias && !this.aliasAndUrl[result.alias]) {

package/lib/support/util.js

@@ -38,6 +38,7 @@ export function logResultLogLine(results) {
       fcp = '',
       cls = '',
       tbt = '',
+      cpuBenchmark = '',
       nRuns = result.browserScripts.length,
       m = nRuns > 1;
     if (results.har && results.har.log.pages[index]) {
@@ -59,6 +60,16 @@ export function logResultLogLine(results) {
           : totalSize + ' bytes';
     }
 
+    if (result.statistics.browser && result.statistics.browser.cpuBenchmark) {
+      cpuBenchmark = formatMetric(
+        'CPUBenchmark',
+        result.statistics.browser.cpuBenchmark,
+        true,
+        true,
+        m
+      );
+    }
+
     if (result.statistics.timings && result.statistics.timings.pageTimings) {
       let pt = result.statistics.timings.pageTimings,
         t = result.statistics.timings,
@@ -178,6 +189,7 @@ export function logResultLogLine(results) {
         lcp,
         cls,
         tbt,
+        cpuBenchmark,
         pageLoad,
         memory,
         speedIndex,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "browsertime",
   "description": "Get performance metrics from your web page using Browsertime.",
-  "version": "17.3.0",
+  "version": "17.5.0-beta1",
   "bin": "./bin/browsertime.js",
   "type": "module",
   "dependencies": {
@@ -12,6 +12,7 @@
     "@sitespeed.io/geckodriver": "0.32.2",
     "@sitespeed.io/throttle": "5.0.0",
     "@sitespeed.io/tracium": "0.3.3",
+    "ff-test-bidi-har-export": "0.0.5",
     "btoa": "1.2.1",
     "chrome-har": "0.13.1",
     "chrome-remote-interface": "0.32.1",

package/lib/firefox/getHAR.js

@@ -1,66 +0,0 @@
-import intel from 'intel';
-const log = intel.getLogger('browsertime.firefox');
-
-// Get the HAR from Firefox
-// Using https://github.com/firefox-devtools/har-export-trigger
-export async function getHAR(runner, includeResponseBodies) {
-  const script = `
-    const callback = arguments[arguments.length - 1];
-    async function triggerExport() {
-      try {
-        const result = await HAR.triggerExport();
-        if (result.pages.length > 0) {
-          result.pages[0].title = document.URL;
-          return callback({'har': {log: result}});
-        } else {
-          return callback({'error': 'The HAR export trigger returned an empty HAR' + JSON.stringify(result)});
-        }
-      }
-      catch(e) {
-        // Sometimes the HAR trigger is really slow so then HAR is not defined.
-        // we catch that with one extra delay
-        const delay = ms => new Promise(res => setTimeout(res, ms));
-        await delay(10000);
-        try {
-          const result = await HAR.triggerExport();
-          result.pages[0].title = document.URL;
-          return callback({'har': {log: result}});
-        } catch(e) {
-          return callback({'error': e.toString()});
-        }
-      }
-    };
-    return triggerExport();
-    `;
-
-  log.info('Waiting on har-export-trigger to collect the HAR');
-  try {
-    const harResult = await runner.runAsyncScript(script, 'GET_HAR_SCRIPT');
-    if (harResult.har) {
-      // The HAR from Firefox always includes content and that can make the HAR file
-      // really big, so we have an option to remove it.
-      if (
-        includeResponseBodies === 'none' ||
-        includeResponseBodies === 'html'
-      ) {
-        for (let entry of harResult.har.log.entries) {
-          if (includeResponseBodies === 'none') {
-            delete entry.response.content.text;
-          } else if (
-            entry.response.content.mimeType &&
-            !entry.response.content.mimeType.includes('text/html')
-          ) {
-            delete entry.response.content.text;
-          }
-        }
-      }
-      return harResult.har;
-    } else {
-      log.error(
-        'Got an error from HAR Export Trigger ' + JSON.stringify(harResult)
-      );
-    }
-  } catch (error) {
-    log.error('Could not get the HAR from Firefox', error);
-  }
-}