browsertime 16.7.0 → 16.9.0

package/lib/chrome/longTaskMetrics.js

@@ -5,6 +5,7 @@ const get = require('lodash.get');
 module.exports = function (result, options) {
   let totalDurationFirstPaint = 0;
   let totalDurationFirstContentFulPaint = 0;
+  let totalDurationLargestContentFulPaint = 0;
   let totalDurationAfterLoadEventEnd = 0;
   let totalDuration = 0;
 
@@ -22,10 +23,16 @@ module.exports = function (result, options) {
     'browserScripts.timings.paintTiming.first-contentful-paint'
   );
 
+  const largestContentfulPaint = get(
+    result,
+    'browserScripts.timings.largestContentfulPaint.renderTime'
+  ); // result.browserScripts.timings.largestContentfulPaint.loadTime;
+
   const loadEventEnd = get(result, 'browserScripts.timings.loadEventEnd');
 
   let longTasksBeforeFirstPaint = 0;
   let longTasksBeforeFirstContentfulPaint = 0;
+  let longTasksBeforeLargestContentfulPaint = 0;
   let longTasksAfterLoadEventEnd = 0;
   let lastLongTask = 0;
 
@@ -42,6 +49,10 @@ module.exports = function (result, options) {
       longTasksBeforeFirstContentfulPaint++;
       totalDurationFirstContentFulPaint += longTask.duration;
     }
+    if (largestContentfulPaint && longTask.startTime < largestContentfulPaint) {
+      longTasksBeforeLargestContentfulPaint++;
+      totalDurationLargestContentFulPaint += longTask.duration;
+    }
     if (
       firstContentfulPaint &&
       longTask.startTime > firstContentfulPaint &&
@@ -74,6 +85,10 @@ module.exports = function (result, options) {
         tasks: longTasksBeforeFirstContentfulPaint,
         totalDuration: totalDurationFirstContentFulPaint
       },
+      beforeLargestContentfulPaint: {
+        tasks: longTasksBeforeLargestContentfulPaint,
+        totalDuration: totalDurationLargestContentFulPaint
+      },
       afterLoadEventEnd: {
         tasks: longTasksAfterLoadEventEnd,
         totalDuration: totalDurationAfterLoadEventEnd

package/lib/chrome/webdriver/setupChromiumOptions.js

@@ -99,6 +99,10 @@ module.exports = function (seleniumOptions, browserOptions, options, baseDir) {
     }
   }
 
+  if (options.debug) {
+    seleniumOptions.addArguments('--auto-open-devtools-for-tabs');
+  }
+
   const perfLogConf = { enableNetwork: true, enablePage: true };
   seleniumOptions.setPerfLoggingPrefs(perfLogConf);
 

package/lib/core/engine/command/debug.js

@@ -0,0 +1,48 @@
+'use strict';
+
+const log = require('intel').getLogger('browsertime.command.debug');
+const delay = ms => new Promise(res => setTimeout(res, ms));
+
+class Debug {
+  constructor(browser, options) {
+    this.browser = browser;
+    this.options = options;
+  }
+
+  /**
+   * Add a breakpoint to script. The browser will wait at the breakpoint for user input.
+   * @returns {Promise} Promise object that is fulfilled when the user move on from the breakpoint.
+   */
+  async breakpoint(name) {
+    if (this.options.debug) {
+      const logMessage = `Pausing for breakpoint ${
+        name || ''
+      }, set browsertime.pause = false; to continue`;
+
+      await this.browser.runScript(
+        `console.log('${logMessage}');`,
+        'CONSOLE_LOG'
+      );
+      log.info(logMessage);
+      let debug = true;
+      await this.browser.runScript(
+        'window.browsertime = {}; window.browsertime.pause = true',
+        'PAUSE_SCRIPT'
+      );
+      while (debug === true) {
+        debug = await this.browser.runScript(
+          'return window.browsertime.pause',
+          'GET_PAUSE_STATUS'
+        );
+        await delay(1000);
+      }
+      log.info('Exit breakpoint and continue.');
+      return this.browser.runScript(
+        'window.browsertime.pause = false',
+        'SET_PAUSE_STATUS'
+      );
+    }
+  }
+}
+
+module.exports = Debug;

package/lib/core/engine/command/wait.js

@@ -101,5 +101,28 @@ class Wait {
   async byPageToComplete() {
     return this.browser.extraWait(this.pageCompleteCheck);
   }
+
+  /**
+   * Wait for an condition that will eventually return a truthy-value for maxTime.
+   * @param {string} jsExpression The js code condition to wait for
+   * @param {number} maxTime Max time to wait in ms
+   * @returns {Promise} Promise object represents when the expression becomes truthy or the time times out
+   * @throws Will throw an error if the condition returned false
+   */
+  async byCondition(jsExpression, maxTime) {
+    const driver = this.browser.getDriver();
+    const time = maxTime || 6000;
+
+    try {
+      const script = `return ${jsExpression}`;
+      await driver.wait(() => {
+        return driver.executeScript(script);
+      }, time);
+    } catch (e) {
+      log.error('Condition was not met', jsExpression, time);
+      log.verbose(e);
+      throw Error(`Condition ${jsExpression} was not met in ${time} ms`);
+    }
+  }
 }
 module.exports = Wait;

package/lib/core/engine/iteration.js

@@ -22,6 +22,7 @@ const Cache = require('./command/cache.js');
 const Meta = require('./command/meta.js');
 const StopWatch = require('./command/stopWatch.js');
 const Select = require('./command/select.js');
+const Debug = require('./command/debug.js');
 const AndroidCommand = require('./command/android.js');
 const ChromeDevToolsProtocol = require('./command/chromeDevToolsProtocol.js');
 const { addConnectivity, removeConnectivity } = require('../../connectivity');
@@ -148,7 +149,7 @@ class Iteration {
 
       const cdp = new ChromeDevToolsProtocol(engineDelegate, options.browser);
       const android = new Android(options);
-
+      const debug = new Debug(browser, options);
       const commands = {
         click: new Click(browser, this.pageCompleteCheck),
         scroll: new Scroll(browser, options),
@@ -172,6 +173,7 @@ class Iteration {
         screenshot: new Screenshot(screenshotManager, browser, index),
         cdp,
         android: new AndroidCommand(options),
+        debug: debug,
         mouse: {
           moveTo: new MouseMove(browser),
           contextClick: new ContextClick(browser),
@@ -244,7 +246,17 @@ class Iteration {
       }
 
       await engineDelegate.beforeBrowserStop(browser, index, result);
-      await browser.stop();
+
+      // if we are in debig mode we wait on a continue command
+      if (options.debug) {
+        await debug.breakpoint('End-of-iteration');
+      }
+
+      if (options.browser === 'firefox' && options.debug) {
+        log.info('Firefox is kept open in debug mode');
+      } else {
+        await browser.stop();
+      }
       // Give the browsers some time to stop
       await delay(2000);
       await engineDelegate.afterBrowserStopped();
@@ -329,7 +341,11 @@ class Iteration {
         if (options.connectivity.variance) {
           await removeConnectivity(options);
         }
-        await browser.stop();
+        if (options.browser === 'firefox' && options.debug) {
+          log.info('Firefox is kept open in debug mode');
+        } else {
+          await browser.stop();
+        }
       } catch (e) {
         // Most cases the browser been stopped already
       }

package/lib/core/seleniumRunner.js

@@ -5,7 +5,11 @@ const merge = require('lodash.merge');
 const get = require('lodash.get');
 const { Condition } = require('selenium-webdriver/lib/webdriver');
 const { isAndroidConfigured } = require('../android');
-const { UrlLoadError, BrowserError } = require('../support/errors');
+const {
+  UrlLoadError,
+  BrowserError,
+  TimeoutError
+} = require('../support/errors');
 const builder = require('./webdriver');
 const getViewPort = require('../support/getViewPort');
 const defaultPageCompleteCheck = require('./pageCompleteChecks/defaultPageCompleteCheck');
@@ -18,7 +22,7 @@ const defaults = {
     pageLoad: 300000,
     script: 120000,
     logs: 90000,
-    pageCompleteCheck: 300000
+    pageCompleteCheck: 60000
   },
   index: 0
 };
@@ -36,7 +40,7 @@ async function timeout(promise, ms, errorMessage) {
 
   return Promise.race([
     new Promise((resolve, reject) => {
-      timer = setTimeout(reject, ms, new BrowserError(errorMessage));
+      timer = setTimeout(reject, ms, new TimeoutError(errorMessage));
       return timer;
     }),
     promise.then(value => {
@@ -182,20 +186,26 @@ class SeleniumRunner {
         `Running page complete check ${pageCompleteCheck} took too long`
       );
     } catch (e) {
-      log.error(
-        `Failed waiting on page ${
-          url ? url : ''
-        } to finished loading, timed out after ${pageCompleteCheckTimeout} ms`,
-        e
-      );
-      throw new UrlLoadError(
-        `Failed waiting on page ${
-          url ? url : ''
-        }  to finished loading, timed out after ${pageCompleteCheckTimeout} ms `,
-        {
-          cause: e
-        }
-      );
+      if (e instanceof TimeoutError) {
+        log.info(
+          `The page did not finishied loading in ${pageCompleteCheckTimeout} ms. You can adjust the timeout by setting the --maxLoadTime option (in ms).`
+        );
+      } else {
+        log.error(
+          `Failed waiting on page ${
+            url ? url : ''
+          } to finished loading, timed out after ${pageCompleteCheckTimeout} ms`,
+          e
+        );
+        throw new UrlLoadError(
+          `Failed waiting on page ${
+            url ? url : ''
+          }  to finished loading, timed out after ${pageCompleteCheckTimeout} ms `,
+          {
+            cause: e
+          }
+        );
+      }
     }
   }
 

package/lib/firefox/webdriver/builder.js

@@ -103,6 +103,12 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
     });
   }
 
+  if (options.debug) {
+    ffOptions.addArguments('--devtools');
+    // Mozilla use detach to make sure you can inspect the browser
+    ffOptions.setPreference('detach', true);
+  }
+
   if (!options.skipHar) {
     if (options.android) {
       log.info('Skip install HAR trigger on Android');
@@ -110,8 +116,10 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
       // Hack for opening the toolbar
       // In Firefox 61 we need to have devtools open but do not need to choose netmonitor
       // ffOptions.setPreference('devtools.toolbox.selectedTool', 'netmonitor');
-      ffOptions.setPreference('devtools.toolbox.footer.height', 0);
-      ffOptions.addArguments('-devtools');
+      if (!options.debug) {
+        ffOptions.setPreference('devtools.toolbox.footer.height', 0);
+        ffOptions.addArguments('-devtools');
+      }
 
       ffOptions.addExtensions(
         path.resolve(

package/lib/firefox/webdriver/firefox.js

@@ -104,19 +104,6 @@ class Firefox {
    */
   async afterPageCompleteCheck(runner, index, url, alias) {
     const result = { url, alias };
-    if (this.firefoxConfig.collectMozLog) {
-      const files = await fileUtil.findFiles(this.baseDir, 'moz_log.txt');
-      for (const file of files) {
-        await rename(
-          `${this.baseDir}/${file}`,
-          path.join(
-            this.baseDir,
-            pathToFolder(result.url, this.options),
-            `${file}-${index}.txt`
-          )
-        );
-      }
-    }
 
     if (this.android && this.options.androidPower) {
       result.power = await this.android.measurePowerUsage(
@@ -192,6 +179,20 @@ class Firefox {
 
   async postWork(index, results) {
     for (let i = 0; i < results.length; i++) {
+      if (this.firefoxConfig.collectMozLog) {
+        const files = await fileUtil.findFiles(this.baseDir, 'moz_log.txt');
+        for (const file of files) {
+          await rename(
+            `${this.baseDir}/${file}`,
+            path.join(
+              this.baseDir,
+              pathToFolder(results[i].url, this.options),
+              `${file}-${index}.txt`
+            )
+          );
+        }
+      }
+
       if (this.firefoxConfig.geckoProfiler && this.options.visualMetrics) {
         const profileFilename = `geckoProfile-${index}.json.gz`;
         const profileSubdir = pathToFolder(results[i].url, this.options);

package/lib/support/cli.js

@@ -75,6 +75,18 @@ function validateInput(argv) {
     return 'You need to specify the --safari.deviceUDID when you run the simulator.';
   }
 
+  if (argv.debug && argv.browser === 'safari') {
+    return 'Debug mode do not work in Safari. Please try with Firefox/Chrome or Edge';
+  }
+
+  if (argv.debug && argv.android) {
+    return 'Debug mode do not work on Android. Please run debug mode on Desktop.';
+  }
+
+  if (argv.debug && argv.docker) {
+    return 'There is no benefit running debug mode inside a Docker container.';
+  }
+
   if (argv.tcpdump) {
     if (!hasbin.all.sync(['tcpdump'])) {
       return 'You need to have tcpdump in your path to be able to record a tcpdump.';
@@ -150,7 +162,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
       group: 'timeouts'
     })
     .option('timeouts.pageCompleteCheck', {
-      default: 300000,
+      alias: 'maxLoadTime',
+      default: 120000,
       type: 'number',
       describe:
         'Timeout when waiting for page to complete loading, in milliseconds',
@@ -1159,6 +1172,12 @@ module.exports.parseCommandLine = function parseCommandLine() {
       describe:
         'The wait time before you start the real testing after your pre-cache request.'
     })
+    .option('debug', {
+      type: 'boolean',
+      default: false,
+      describe: 'Run Browsertime in debug mode.',
+      group: 'debug'
+    })
 
     .count('verbose')
     .config(config)
@@ -1253,6 +1272,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
     set(argv, 'visualMetrics', get(argv, 'visualMetrics', true));
   }
 
+  // Set the timeouts to a maximum while debugging
+  if (argv.debug) {
+    set(argv, 'timeouts.pageload', 2147483647);
+    set(argv, 'timeouts.script', 2147483647);
+    set(argv, 'timeouts.pageCompleteCheck', 2147483647);
+  }
+
   return {
     urls: argv._,
     options: argv

package/lib/support/errors.js

@@ -21,8 +21,16 @@ class UrlLoadError extends BrowsertimeError {
   }
 }
 
+class TimeoutError extends BrowsertimeError {
+  constructor(message, url, extra) {
+    super(message, extra);
+    this.url = url;
+  }
+}
+
 module.exports = {
   BrowsertimeError,
   BrowserError,
-  UrlLoadError
+  UrlLoadError,
+  TimeoutError
 };

package/lib/video/postprocessing/visualmetrics/visualMetrics.js

@@ -81,10 +81,6 @@ module.exports = {
       scriptArgs.push('--perceptual');
     }
 
-    if (options.visualMetricsNotification) {
-      scriptArgs.push('--notification');
-    }
-
     if (options.visualMetricsContentful) {
       scriptArgs.push('--contentful');
     }

package/package.json

@@ -1,11 +1,11 @@
 {
   "description": "Browsertime",
-  "version": "16.7.0",
+  "version": "16.9.0",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@cypress/xvfb": "1.2.4",
     "@devicefarmer/adbkit": "2.11.3",
-    "@sitespeed.io/chromedriver": "101.0.4951-15",
+    "@sitespeed.io/chromedriver": "102.0.5005-27",
     "@sitespeed.io/edgedriver": "101.0.1210-32",
     "@sitespeed.io/geckodriver": "0.31.0",
     "@sitespeed.io/throttle": "3.1.1",
@@ -26,7 +26,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.1.2",
+    "selenium-webdriver": "4.2.0",
     "yargs": "17.4.1"
   },
   "optionalDependencies": {