browsertime 16.5.0 → 16.8.0

package/CHANGELOG.md

@@ -1,6 +1,23 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
-##  16.5.0 - 2022-05-11
+##  16.8.0 - 2022-06-06
+### Added
+* Add `--debug` mode. Debug mode will run your tests and open devtools in Chrome/Edge/Firefox on desktop and will stop after each iteration so you can inspect the page. You can add your own breakpoint in your script with the `breakpoint(name)` command. To continue after your breakpoint, add the following code in the developer console in your browser: `window.browsertime.pause=false;`  [#1798](https://github.com/sitespeedio/browsertime/pull/1798).
+* Use Selenium WebDriver 4.2.0 [#1801](https://github.com/sitespeedio/browsertime/pull/1801).
+* Updated to Firefox 101, Edge 102 in the Docker container.
+
+### Tech
+* Added tests using NodeJS 18 [#1772](https://github.com/sitespeedio/browsertime/pull/1772).
+## 16.7.0 - 2022-05-20
+### Added
+* Include last CPU long task in the HAR file so we can show when it happens [#1793](https://github.com/sitespeedio/browsertime/pull/1793).
+* Inlclude TTFB and INP in the Google Web Vital namespace [#1792](https://github.com/sitespeedio/browsertime/pull/1792).
+## 16.6.0 - 2022-05-20
+### Added
+* Implemented experimental Interaction to next paint that's useful if you test user journeys [#1791](https://github.com/sitespeedio/browsertime/pull/1791).
+* Track when the last CPU long task happen as explained by Andy Davies of the webperf Slack channel [#1789](https://github.com/sitespeedio/browsertime/pull/1789).
+
+## 16.5.0 - 2022-05-11
 ### Added
 * Make it possible to configure the max size in pixels for the filmstrip screenshots using `--videoParams.thumbsize` [#1787](https://github.com/sitespeedio/browsertime/pull/1787).
 

package/bin/browsertime.js

@@ -43,6 +43,9 @@ async function preWarmServer(urls, options) {
 }
 
 async function run(urls, options) {
+  if (options.debug) {
+    log.info('Running Browsertime in debug mode.');
+  }
   try {
     if (!options.resultDir) {
       let dir = 'browsertime-results';

package/browserscripts/pageinfo/interactionToNextPaintInfo.js

@@ -0,0 +1,117 @@
+(function () {
+  // This is an updated version of
+  // https://github.com/GoogleChrome/web-vitals/blob/next/src/onINP.ts
+
+  const supported = PerformanceObserver.supportedEntryTypes;
+  if (!supported || supported.indexOf('event') === -1) {
+    return;
+  }
+
+  const observer = new PerformanceObserver(list => {});
+  // Event Timing entries have their durations rounded to the nearest 8ms,
+  // so a duration of 40ms would be any event that spans 2.5 or more frames
+  // at 60Hz. This threshold is chosen to strike a balance between usefulness
+  // and performance. Running this callback for any interaction that spans
+  // just one or two frames is likely not worth the insight that could be
+  // gained.
+  observer.observe({type: 'event', buffered: true, durationThreshold: 40});
+  const entries = observer.takeRecords();
+
+  // To prevent unnecessary memory usage on pages with lots of interactions,
+  // store at most 10 of the longest interactions to consider as INP candidates.
+  const MAX_INTERACTIONS_TO_CONSIDER = 10;
+
+  // A list of longest interactions on the page (by latency) sorted so the
+  // longest one is first. The list is as most MAX_INTERACTIONS_TO_CONSIDER long.
+  let longestInteractionList = [];
+
+  // A mapping of longest interactions by their interaction ID.
+  // This is used for faster lookup.
+  const longestInteractionMap = {};
+
+  const getInteractionCountForNavigation = () => {
+    // I guess the interactionCount is coming in Chrome later on
+    if (performance.interactionCount) {
+      return performance.interactionCount;
+    } else {
+      const observerForAll = new PerformanceObserver(list => {});
+      observerForAll.observe({
+        type: 'event',
+        buffered: true,
+        durationThreshold: 0
+      });
+      const allEntries = observerForAll.takeRecords();
+      let interactionCountEstimate = 0;
+      let minKnownInteractionId = Infinity;
+      let maxKnownInteractionId = 0;
+
+      for (let e of allEntries) {
+        if (e.interactionId) {
+          minKnownInteractionId = Math.min(
+            minKnownInteractionId,
+            e.interactionId
+          );
+          maxKnownInteractionId = Math.max(
+            maxKnownInteractionId,
+            e.interactionId
+          );
+
+          interactionCountEstimate = maxKnownInteractionId
+            ? (maxKnownInteractionId - minKnownInteractionId) / 7 + 1
+            : 0;
+        }
+      }
+      return interactionCountEstimate;
+    }
+  };
+
+  const estimateP98LongestInteraction = () => {
+    const candidateInteractionIndex = Math.min(
+      longestInteractionList.length - 1,
+      Math.floor(getInteractionCountForNavigation() / 50)
+    );
+
+    return longestInteractionList[candidateInteractionIndex];
+  };
+
+  if (entries.length > 0) {
+    for (let entry of entries) {
+      const minLongestInteraction =
+        longestInteractionList[longestInteractionList.length - 1];
+      const existingInteraction = longestInteractionMap[entry.interactionId];
+      if (
+        existingInteraction ||
+        longestInteractionList.length < MAX_INTERACTIONS_TO_CONSIDER ||
+        entry.duration > minLongestInteraction.latency
+      ) {
+        // If the interaction already exists, update it. Otherwise create one.
+        if (existingInteraction) {
+          existingInteraction.entries.push(entry);
+          existingInteraction.latency = Math.max(
+            existingInteraction.latency,
+            entry.duration
+          );
+        } else {
+          const interaction = {
+            id: entry.interactionId,
+            latency: entry.duration,
+            entries: [entry]
+          };
+          longestInteractionMap[interaction.id] = interaction;
+          longestInteractionList.push(interaction);
+        }
+
+        // Sort the entries by latency (descending) and keep only the top ten.
+        longestInteractionList.sort((a, b) => b.latency - a.latency);
+        longestInteractionList
+          .splice(MAX_INTERACTIONS_TO_CONSIDER)
+          .forEach(i => {
+            delete longestInteractionMap[i.id];
+          });
+      }
+    }
+    return estimateP98LongestInteraction();
+  } else {
+    // nothing to report
+  }
+})();

package/browserscripts/timings/interactionToNextPaint.js

@@ -0,0 +1,116 @@
+(function () {
+  // This is an updated version of
+  // https://github.com/GoogleChrome/web-vitals/blob/next/src/onINP.ts
+
+  const supported = PerformanceObserver.supportedEntryTypes;
+  if (!supported || supported.indexOf('event') === -1) {
+    return;
+  }
+
+  // To prevent unnecessary memory usage on pages with lots of interactions,
+  // store at most 10 of the longest interactions to consider as INP candidates.
+  const MAX_INTERACTIONS_TO_CONSIDER = 10;
+
+  // A list of longest interactions on the page (by latency) sorted so the
+  // longest one is first. The list is as most MAX_INTERACTIONS_TO_CONSIDER long.
+  let longestInteractionList = [];
+
+  // A mapping of longest interactions by their interaction ID.
+  // This is used for faster lookup.
+  const longestInteractionMap = {};
+
+  const getInteractionCountForNavigation = () => {
+    // I guess the interactionCount is coming in Chrome later on
+    if (performance.interactionCount) {
+      return performance.interactionCount;
+    } else {
+      const observerForAll = new PerformanceObserver(list => {});
+      observerForAll.observe({
+        type: 'event',
+        buffered: true,
+        durationThreshold: 0
+      });
+      const allEntries = observerForAll.takeRecords();
+      let interactionCountEstimate = 0;
+      let minKnownInteractionId = Infinity;
+      let maxKnownInteractionId = 0;
+
+      for (let e of allEntries) {
+        if (e.interactionId) {
+          minKnownInteractionId = Math.min(
+            minKnownInteractionId,
+            e.interactionId
+          );
+          maxKnownInteractionId = Math.max(
+            maxKnownInteractionId,
+            e.interactionId
+          );
+
+          interactionCountEstimate = maxKnownInteractionId
+            ? (maxKnownInteractionId - minKnownInteractionId) / 7 + 1
+            : 0;
+        }
+      }
+      return interactionCountEstimate;
+    }
+  };
+
+  const estimateP98LongestInteraction = () => {
+    const candidateInteractionIndex = Math.min(
+      longestInteractionList.length - 1,
+      Math.floor(getInteractionCountForNavigation() / 50)
+    );
+
+    return longestInteractionList[candidateInteractionIndex];
+  };
+
+  const observer = new PerformanceObserver(list => {});
+  // Event Timing entries have their durations rounded to the nearest 8ms,
+  // so a duration of 40ms would be any event that spans 2.5 or more frames
+  // at 60Hz. This threshold is chosen to strike a balance between usefulness
+  // and performance. Running this callback for any interaction that spans
+  // just one or two frames is likely not worth the insight that could be
+  // gained.
+  observer.observe({type: 'event', buffered: true, durationThreshold: 40});
+  const entries = observer.takeRecords();
+
+  if (entries.length > 0) {
+    for (let entry of entries) {
+      const minLongestInteraction =
+        longestInteractionList[longestInteractionList.length - 1];
+      const existingInteraction = longestInteractionMap[entry.interactionId];
+      if (
+        existingInteraction ||
+        longestInteractionList.length < MAX_INTERACTIONS_TO_CONSIDER ||
+        entry.duration > minLongestInteraction.latency
+      ) {
+        // If the interaction already exists, update it. Otherwise create one.
+        if (existingInteraction) {
+          existingInteraction.latency = Math.max(
+            existingInteraction.latency,
+            entry.duration
+          );
+        } else {
+          const interaction = {
+            id: entry.interactionId,
+            latency: entry.duration
+          };
+          longestInteractionMap[interaction.id] = interaction;
+          longestInteractionList.push(interaction);
+        }
+
+        // Sort the entries by latency (descending) and keep only the top ten.
+        longestInteractionList.sort((a, b) => b.latency - a.latency);
+        longestInteractionList
+          .splice(MAX_INTERACTIONS_TO_CONSIDER)
+          .forEach(i => {
+            delete longestInteractionMap[i.id];
+          });
+      }
+    }
+    const inp = estimateP98LongestInteraction();
+    return inp.latency;
+  } else {
+    // nothing to report
+  }
+})();

package/lib/chrome/longTaskMetrics.js

@@ -27,9 +27,13 @@ module.exports = function (result, options) {
   let longTasksBeforeFirstPaint = 0;
   let longTasksBeforeFirstContentfulPaint = 0;
   let longTasksAfterLoadEventEnd = 0;
+  let lastLongTask = 0;
 
   for (let longTask of result.browserScripts.pageinfo.longTask) {
     totalDuration += longTask.duration;
+    if (longTask.startTime > lastLongTask) {
+      lastLongTask = longTask.startTime;
+    }
     if (firstPaint && longTask.startTime < firstPaint) {
       longTasksBeforeFirstPaint++;
       totalDurationFirstPaint += longTask.duration;
@@ -61,6 +65,7 @@ module.exports = function (result, options) {
       totalDuration,
       totalBlockingTime: Number(totalBlockingTime.toFixed(0)),
       maxPotentialFid: Number(maxPotentialFid.toFixed(0)),
+      lastLongTask: Number(lastLongTask.toFixed(0)),
       beforeFirstPaint: {
         tasks: longTasksBeforeFirstPaint,
         totalDuration: totalDurationFirstPaint

package/lib/chrome/webdriver/chromium.js

@@ -320,6 +320,10 @@ class Chromium {
         result.browserScripts.pageinfo.cumulativeLayoutShift;
     }
     if (result.browserScripts.timings) {
+      if (result.browserScripts.timings.ttfb) {
+        result.googleWebVitals.ttfb = result.browserScripts.timings.ttfb;
+      }
+
       if (result.browserScripts.timings.largestContentfulPaint) {
         result.googleWebVitals.largestContentfulPaint =
           result.browserScripts.timings.largestContentfulPaint.renderTime ||
@@ -338,6 +342,11 @@ class Chromium {
           : 0;
       }
 
+      if (result.browserScripts.timings.interactionToNextPaint) {
+        result.googleWebVitals.interactionToNextPaint =
+          result.browserScripts.timings.interactionToNextPaint;
+      }
+
       // Add LCP to the HAR
       if (
         result.browserScripts.timings.largestContentfulPaint &&

package/lib/chrome/webdriver/setupChromiumOptions.js

@@ -99,6 +99,10 @@ module.exports = function (seleniumOptions, browserOptions, options, baseDir) {
     }
   }
 
+  if (options.debug) {
+    seleniumOptions.addArguments('--auto-open-devtools-for-tabs');
+  }
+
   const perfLogConf = { enableNetwork: true, enablePage: true };
   seleniumOptions.setPerfLoggingPrefs(perfLogConf);
 

package/lib/core/engine/command/debug.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const log = require('intel').getLogger('browsertime.command.debug');
+const delay = ms => new Promise(res => setTimeout(res, ms));
+
+class Debug {
+  constructor(browser, options) {
+    this.browser = browser;
+    this.options = options;
+  }
+
+  /**
+   * Add a breakpoint to script. The browser will wait at the breakpoint for user input.
+   * @returns {Promise} Promise object that is fulfilled when the user move on from the breakpoint.
+   */
+  async breakpoint(name) {
+    if (this.options.debug) {
+      const logMessage = `Pausing for breakpoint ${
+        name || ''
+      }, set browsertime.pause = false; to continue`;
+
+      await this.browser.runScript(
+        `console.log('${logMessage}');`,
+        'CONSOLE_LOG'
+      );
+      log.info(logMessage);
+      let debug = true;
+      await this.browser.runScript(
+        'window.browsertime = {}; window.browsertime.pause = true',
+        'PAUSE_SCRIPT'
+      );
+      while (debug === true) {
+        debug = await this.browser.runScript(
+          'return window.browsertime.pause',
+          'GET_PAUSE_STATUS'
+        );
+        await delay(1000);
+      }
+      return this.browser.runScript(
+        'window.browsertime.pause = false',
+        'SET_PAUSE_STATUS'
+      );
+    }
+  }
+}
+
+module.exports = Debug;

package/lib/core/engine/iteration.js

@@ -22,6 +22,7 @@ const Cache = require('./command/cache.js');
 const Meta = require('./command/meta.js');
 const StopWatch = require('./command/stopWatch.js');
 const Select = require('./command/select.js');
+const Debug = require('./command/debug.js');
 const AndroidCommand = require('./command/android.js');
 const ChromeDevToolsProtocol = require('./command/chromeDevToolsProtocol.js');
 const { addConnectivity, removeConnectivity } = require('../../connectivity');
@@ -148,7 +149,7 @@ class Iteration {
 
       const cdp = new ChromeDevToolsProtocol(engineDelegate, options.browser);
       const android = new Android(options);
-
+      const debug = new Debug(browser, options);
       const commands = {
         click: new Click(browser, this.pageCompleteCheck),
         scroll: new Scroll(browser, options),
@@ -172,6 +173,7 @@ class Iteration {
         screenshot: new Screenshot(screenshotManager, browser, index),
         cdp,
         android: new AndroidCommand(options),
+        debug: debug,
         mouse: {
           moveTo: new MouseMove(browser),
           contextClick: new ContextClick(browser),
@@ -244,7 +246,17 @@ class Iteration {
       }
 
       await engineDelegate.beforeBrowserStop(browser, index, result);
-      await browser.stop();
+
+      // if we are in debig mode we wait on a continue command
+      if (options.debug) {
+        await debug.breakpoint('End-of-iteration');
+      }
+
+      if (options.browser === 'firefox' && options.debug) {
+        log.info('Firefox is kept open in debug mode');
+      } else {
+        await browser.stop();
+      }
       // Give the browsers some time to stop
       await delay(2000);
       await engineDelegate.afterBrowserStopped();
@@ -329,7 +341,11 @@ class Iteration {
         if (options.connectivity.variance) {
           await removeConnectivity(options);
         }
-        await browser.stop();
+        if (options.browser === 'firefox' && options.debug) {
+          log.info('Firefox is kept open in debug mode');
+        } else {
+          await browser.stop();
+        }
       } catch (e) {
         // Most cases the browser been stopped already
       }

package/lib/firefox/webdriver/builder.js

@@ -103,6 +103,12 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
     });
   }
 
+  if (options.debug) {
+    ffOptions.addArguments('--devtools');
+    // Mozilla use detach to make sure you can inspect the browser
+    ffOptions.setPreference('detach', true);
+  }
+
   if (!options.skipHar) {
     if (options.android) {
       log.info('Skip install HAR trigger on Android');
@@ -110,8 +116,10 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
       // Hack for opening the toolbar
       // In Firefox 61 we need to have devtools open but do not need to choose netmonitor
       // ffOptions.setPreference('devtools.toolbox.selectedTool', 'netmonitor');
-      ffOptions.setPreference('devtools.toolbox.footer.height', 0);
-      ffOptions.addArguments('-devtools');
+      if (!options.debug) {
+        ffOptions.setPreference('devtools.toolbox.footer.height', 0);
+        ffOptions.addArguments('-devtools');
+      }
 
       ffOptions.addExtensions(
         path.resolve(

package/lib/support/cli.js

@@ -75,6 +75,18 @@ function validateInput(argv) {
     return 'You need to specify the --safari.deviceUDID when you run the simulator.';
   }
 
+  if (argv.debug && argv.browser === 'safari') {
+    return 'Debug mode do not work in Safari. Please try with Firefox/Chrome or Edge';
+  }
+
+  if (argv.debug && argv.android) {
+    return 'Debug mode do not work on Android. Please run debug mode on Desktop.';
+  }
+
+  if (argv.debug && argv.docker) {
+    return 'There is no benefit running debug mode inside a Docker container.';
+  }
+
   if (argv.tcpdump) {
     if (!hasbin.all.sync(['tcpdump'])) {
       return 'You need to have tcpdump in your path to be able to record a tcpdump.';
@@ -1159,6 +1171,12 @@ module.exports.parseCommandLine = function parseCommandLine() {
       describe:
         'The wait time before you start the real testing after your pre-cache request.'
     })
+    .option('debug', {
+      type: 'boolean',
+      default: false,
+      describe: 'Run Browsertime in debug mode.',
+      group: 'debug'
+    })
 
     .count('verbose')
     .config(config)
@@ -1253,6 +1271,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
     set(argv, 'visualMetrics', get(argv, 'visualMetrics', true));
   }
 
+  // Set the timeouts to a maximum while debugging
+  if (argv.debug) {
+    set(argv, 'timeouts.pageload', 2147483647);
+    set(argv, 'timeouts.script', 2147483647);
+    set(argv, 'timeouts.pageCompleteCheck', 2147483647);
+  }
+
   return {
     urls: argv._,
     options: argv

package/lib/support/har/index.js

@@ -91,6 +91,10 @@ function addExtrasToHAR(
     harPageTimings._firstPaint = timings.firstPaint;
   }
 
+  if (cpu && cpu.longTasks && cpu.longTasks.lastLongTask) {
+    harPageTimings._lastCPULongTask = cpu.longTasks.lastLongTask;
+  }
+
   if (timings && timings.largestContentfulPaint) {
     harPageTimings._largestContentfulPaint =
       timings.largestContentfulPaint.renderTime;

package/package.json

@@ -1,11 +1,11 @@
 {
   "description": "Browsertime",
-  "version": "16.5.0",
+  "version": "16.8.0",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@cypress/xvfb": "1.2.4",
     "@devicefarmer/adbkit": "2.11.3",
-    "@sitespeed.io/chromedriver": "101.0.4951-15",
+    "@sitespeed.io/chromedriver": "102.0.5005-27",
     "@sitespeed.io/edgedriver": "101.0.1210-32",
     "@sitespeed.io/geckodriver": "0.31.0",
     "@sitespeed.io/throttle": "3.1.1",
@@ -26,7 +26,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.1.2",
+    "selenium-webdriver": "4.2.0",
     "yargs": "17.4.1"
   },
   "optionalDependencies": {