browsertime 17.17.0 → 17.18.1

package/CHANGELOG.md

@@ -1,10 +1,28 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 17.18.1 - 2022-10-31
+### Fixed
+* WebDriver 4.14.0
+* Chromedriver 119.0.6045-59
+
+## 17.18.0 - 2022-10-23
+### Added
+* Updated to Chromedriver 119 [#2003](https://github.com/sitespeedio/browsertime/pull/2003). 119 works with both Chrome 118 and 119 so it fixes [#1197](https://github.com/sitespeedio/browsertime/issues/1997).
+* Add support for network idle method to know when to end a test that uses network logs. Uses Bidi for Firefox and CDP for Chrome to listen on network events to know when to end a test. By default 5 seconds idle network time ends a tests (you could have network responses that hasn't arrived yet) [#1960](https://github.com/sitespeedio/browsertime/pull/1960). Potentially this can help SPA users or users where the page uses iframes. You can try it out by adding `--pageCompleteCheckNetworkIdle` yo your command line. This is still some work in progress but feel free to try ut out.
+* The resources script now collects number of resources served from the browser cashe for browser that supports that through the resource timing API [#1998](https://github.com/sitespeedio/browsertime/pull/1998)  
+
+### Fixed
+* Make sure timer always is cleared. There was case of where we do a rase beteween a promise and a timeout where the timeout timer wasn't cleared/removed [#2005](https://github.com/sitespeedio/browsertime/pull/2005).
+* Better way to get the url when you use GeckoProfiler.stop for Firefox, thank you [Nazım Can Altınova](https://github.com/canova) for PR [#1999](https://github.com/sitespeedio/browsertime/pull/1999)
+
 ## 17.17.0 - 2022-10-11
 ### Added
 * Firefox 118, Edge 117 and Chrome/Chromedriver 118 in the Docker container [#1996](https://github.com/sitespeedio/browsertime/pull/1996).
 * Expose trace start/stop for Chrome in scripting (the same way as for Firefox). Thank you [KS](https://github.com/92kns) for [#1988](https://github.com/sitespeedio/browsertime/pull/1988). Documentation is coming when the functionality is rolled out in sitespeed.io.
 
+### Updated
+* Updated to Selenium 4.12
+
 ## 17.16.0 - 2022-09-04
 ### Added
 * Firefox 117 and Edge 116 in the Docker container.

package/browserscripts/pageinfo/resources.js

@@ -1,14 +1,27 @@
-(function() {
+(function () {
   const resources = window.performance.getEntriesByType('resource');
 
   let resourceDuration = 0;
+  let servedFromCache = 0;
+  let servedFromCacheSupported = false;
   for (let i = 0; i < resources.length; i++) {
     resourceDuration += resources[i].duration;
+    if (resources[i].deliveryType !== undefined) {
+      servedFromCacheSupported = true;
+      if (resources[i].deliveryType === 'cache') {
+        servedFromCache++;
+      }
+    }
   }
 
-  return {
+  const info = {
     count: Number(resources.length),
-    duration: Number(resourceDuration)
+    duration: Number(resourceDuration),
+  };
+
+  if (servedFromCacheSupported === true) {   
+    info.servedFromCache = Number(servedFromCache);
   }
-})();
 
+  return info;
+})();

package/lib/chrome/networkManager.js

@@ -0,0 +1,61 @@
+import intel from 'intel';
+import get from 'lodash.get';
+
+const log = intel.getLogger('browsertime.chrome.network');
+
+export class NetworkManager {
+  constructor(cdpClient, options) {
+    this.maxTimeout = get(options, 'timeouts.pageCompleteCheck', 30_000);
+    this.idleTime = get(options, 'timeouts.networkIdle', 5000);
+
+    this.cdp = cdpClient.getRawClient();
+    this.inflight = 0;
+    this.lastRequestTimestamp;
+    this.lastResponseTimestamp;
+
+    this.cdp.Network.requestWillBeSent(() => {
+      this.inflight++;
+      this.lastRequestTimestamp = Date.now();
+    });
+
+    this.cdp.Network.loadingFinished(() => {
+      this.inflight--;
+      this.lastResponseTimestamp = Date.now();
+    });
+
+    this.cdp.Network.loadingFailed(() => {
+      this.inflight--;
+      this.lastResponseTimestamp = Date.now();
+    });
+  }
+
+  async waitForNetworkIdle() {
+    const startTime = Date.now();
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const now = Date.now();
+      const sinceLastResponseRequest =
+        now - Math.max(this.lastResponseTimestamp, this.lastRequestTimestamp);
+      const sinceStart = now - startTime;
+
+      if (sinceLastResponseRequest >= this.idleTime) {
+        if (this.inflight > 0) {
+          log.info(
+            'Idle time without any request/responses. Inflight requests:' +
+              this.inflight
+          );
+        }
+        break;
+      }
+
+      if (sinceStart >= this.maxTimeout) {
+        log.info(
+          'Timeout waiting for network.  Inflight requests:' + this.inflight
+        );
+        break;
+      }
+
+      await new Promise(r => setTimeout(r, 200));
+    }
+  }
+}

package/lib/chrome/webdriver/chromium.js

@@ -14,6 +14,7 @@ import { logging as _logging } from 'selenium-webdriver';
 import { parse } from '../traceCategoriesParser.js';
 import { pathToFolder } from '../../support/pathToFolder.js';
 import { ChromeDevtoolsProtocol } from '../chromeDevtoolsProtocol.js';
+import { NetworkManager } from '../networkManager.js';
 import { Android, isAndroidConfigured } from '../../android/index.js';
 import { getRenderBlocking } from './traceUtilities.js';
 const unlink = promisify(_unlink);
@@ -502,4 +503,9 @@ export class Chromium {
   async setCookies(url, cookies) {
     return this.cdpClient.setCookies(url, cookies);
   }
+
+  async waitForNetworkIdle() {
+    let network = new NetworkManager(this.cdpClient, this.options);
+    return network.waitForNetworkIdle();
+  }
 }

package/lib/core/engine/command/geckoProfiler.js

@@ -1,11 +1,12 @@
 import intel from 'intel';
 const log = intel.getLogger('browsertime.command.geckoprofiler');
 export class GeckoProfiler {
-  constructor(GeckoProfiler, browser, index, options) {
+  constructor(GeckoProfiler, browser, index, options, result) {
     this.GeckoProfiler = GeckoProfiler;
     this.browser = browser;
     this.index = index;
     this.options = options;
+    this.result = result;
   }
 
   async start() {
@@ -25,11 +26,7 @@ export class GeckoProfiler {
   async stop() {
     if (this.options.browser === 'firefox') {
       if (this.options.firefox.geckoProfilerRecordingType === 'custom') {
-        const url = await this.browser
-          .getDriver()
-          .executeScript('return document.documentURI;');
-
-        return this.GeckoProfiler.stop(this.index, url);
+        return this.GeckoProfiler.stop(this.index, this.result[0].url);
       }
     } else {
       throw new Error('Geckoprofiler only works in Firefox');

package/lib/core/engine/command/measure.js

@@ -154,7 +154,11 @@ export class Measure {
       await this.engineDelegate.beforeStartIteration(this.browser, this.index);
     }
     this.numberOfVisitedPages++;
-    return this.browser.loadAndWait(url, this.pageCompleteCheck);
+    return this.browser.loadAndWait(
+      url,
+      this.pageCompleteCheck,
+      this.engineDelegate
+    );
   }
 
   /**
@@ -229,7 +233,11 @@ export class Measure {
       this.result[this.numberOfMeasuredPages].url = url;
       try {
         await this.engineDelegate.beforeEachURL(this.browser, url);
-        await this.browser.loadAndWait(url, this.pageCompleteCheck);
+        await this.browser.loadAndWait(
+          url,
+          this.pageCompleteCheck,
+          this.engineDelegate
+        );
         return this.stop(url);
       } catch (error) {
         this.result[this.numberOfMeasuredPages].error = [error.name];

package/lib/core/engine/iteration.js

@@ -159,7 +159,8 @@ export class Iteration {
         browserProfiler,
         browser,
         index,
-        this.options
+        this.options,
+        result
       );
       const cdp = new ChromeDevelopmentToolsProtocol(
         engineDelegate,

package/lib/core/seleniumRunner.js

@@ -24,25 +24,40 @@ const defaults = {
 const delay = ms => new Promise(res => setTimeout(res, ms));
 
 /**
- * Timeout a promise after ms. Use promise.race to compete
- * about the timeout and the promise.
- * @param {promise} promise - The promise to wait for
- * @param {int} ms - how long in ms to wait for the promise to fininsh
- * @param {string} errorMessage - the error message in the Error if we timeouts
+ * @function timeout
+ * @description Wraps a promise with a timeout, rejecting the promise with a TimeoutError if it does not settle within the specified time.
+ *
+ * @param {Promise} promise - The promise to wrap with a timeout.
+ * @param {number} ms - The number of milliseconds to wait before timing out.
+ * @param {string} errorMessage - The error message for the TimeoutError.
+ *
+ * @returns {Promise} - A promise that resolves with the value of the input promise if it settles within time, or rejects with a TimeoutError otherwise.
  */
 async function timeout(promise, ms, errorMessage) {
-  let timer;
-
-  return Promise.race([
-    new Promise((resolve, reject) => {
-      timer = setTimeout(reject, ms, new TimeoutError(errorMessage));
-      return timer;
-    }),
-    promise.then(value => {
-      clearTimeout(timer);
-      return value;
-    })
-  ]);
+  let timerId;
+  let finished = false;
+
+  // Create a new promise that rejects after `ms` milliseconds.
+  const timer = new Promise((_, reject) => {
+    timerId = setTimeout(() => {
+      if (!finished) {
+        // Reject with a TimeoutError if the input promise has not yet settled.
+        reject(new Error(errorMessage));
+      }
+    }, ms);
+  });
+
+  try {
+    // Race the input promise against the timer.
+    const result = await Promise.race([promise, timer]);
+    finished = true;
+    clearTimeout(timerId);
+    return result;
+  } catch (error) {
+    finished = true;
+    clearTimeout(timerId);
+    throw error;
+  }
 }
 
 /**
@@ -132,14 +147,15 @@ export class SeleniumRunner {
 
   async extraWait(pageCompleteCheck) {
     await delay(this.options.beforePageCompleteWaitTime || 5000);
-    return this.wait(pageCompleteCheck);
+    return this._waitOnPageCompleteCheck(pageCompleteCheck);
   }
   /**
    * Wait for pageCompleteCheck to end before we return.
    * @param {string} pageCompleteCheck - JavaScript that checks if the page has finished loading
    * @throws {UrlLoadError}
    */
-  async wait(pageCompleteCheck, url) {
+
+  async _waitOnPageCompleteCheck(pageCompleteCheck, url) {
     const waitTime = this.options.pageCompleteWaitTime || 5000;
     if (!pageCompleteCheck) {
       pageCompleteCheck = this.options.pageCompleteCheckInactivity
@@ -209,7 +225,7 @@ export class SeleniumRunner {
    * @param {string} pageCompleteCheck - JavaScript that checks if the page has finished loading
    * @throws {UrlLoadError}
    */
-  async loadAndWait(url, pageCompleteCheck) {
+  async loadAndWait(url, pageCompleteCheck, engine) {
     const driver = this.driver;
     // Browsers may normalize 'https://x.com' differently; in particular, Firefox normalizes to
     // 'https://x.com/'.  This is a first normalization attempt; there are deeper options that order
@@ -263,81 +279,86 @@ export class SeleniumRunner {
       await driver.executeScript(navigate);
     }
 
-    // If you run with default settings, the webdriver will give back
-    // control ASAP. Therefore you want to wait some extra time
-    // before you start to run your page complete check
-    this.options.pageLoadStrategy === 'none'
-      ? await delay(this.options.pageCompleteCheckStartWait || 5000)
-      : await delay(2000);
-
-    // We give it a couple of times to finish loading, this makes it
-    // more stable in real case scenarios on slow servers.
-    let totalWaitTime = 0;
-    const tries = get(this.options, 'retries', 5) + 1;
-    for (let index = 0; index < tries; ++index) {
-      try {
-        await this.wait(pageCompleteCheck, normalizedURI);
-        const newURI = new URL(
-          await driver.executeScript('return document.documentURI;')
-        ).toString();
-        // If we use a SPA it could be that we don't test a new URL so just do one try
-        // and make sure your page complete check take care of other things
-        if (this.options.spa) {
-          break;
-        } else if (normalizedURI === startURI) {
-          // You are navigating to the current page
-          break;
-        } else if (normalizedURI.startsWith('data:text')) {
-          // Navigations between data/text seems to don't change the URI
-          break;
-        } else if (newURI === 'chrome-error://chromewebdata/') {
-          // This is the timeout URL for Chrome, just continue to try
-          throw new UrlLoadError(
-            `Could not load ${url} is the web page down?`,
-            url
-          );
-        } else if (newURI === startURI) {
-          const waitTime = (this.options.retryWaitTime || 10_000) * (index + 1);
-          totalWaitTime += waitTime;
-          log.debug(
-            `URL ${url} failed to load, the ${
-              this.options.browser
-            } are still on ${startURI} , trying ${
-              tries - index - 1
-            } more time(s) but first wait for ${waitTime} ms.`
-          );
+    if (this.options.pageCompleteCheckNetworkIdle) {
+      return engine.waitForNetworkIdle(driver);
+    } else {
+      // If you run with default settings, the webdriver will give back
+      // control ASAP. Therefore you want to wait some extra time
+      // before you start to run your page complete check
+      this.options.pageLoadStrategy === 'none'
+        ? await delay(this.options.pageCompleteCheckStartWait || 5000)
+        : await delay(2000);
+
+      // We give it a couple of times to finish loading, this makes it
+      // more stable in real case scenarios on slow servers.
+      let totalWaitTime = 0;
+      const tries = get(this.options, 'retries', 5) + 1;
+      for (let index = 0; index < tries; ++index) {
+        try {
+          await this._waitOnPageCompleteCheck(pageCompleteCheck, normalizedURI);
+          const newURI = new URL(
+            await driver.executeScript('return document.documentURI;')
+          ).toString();
+          // If we use a SPA it could be that we don't test a new URL so just do one try
+          // and make sure your page complete check take care of other things
+          if (this.options.spa) {
+            break;
+          } else if (normalizedURI === startURI) {
+            // You are navigating to the current page
+            break;
+          } else if (normalizedURI.startsWith('data:text')) {
+            // Navigations between data/text seems to don't change the URI
+            break;
+          } else if (newURI === 'chrome-error://chromewebdata/') {
+            // This is the timeout URL for Chrome, just continue to try
+            throw new UrlLoadError(
+              `Could not load ${url} is the web page down?`,
+              url
+            );
+          } else if (newURI === startURI) {
+            const waitTime =
+              (this.options.retryWaitTime || 10_000) * (index + 1);
+            totalWaitTime += waitTime;
+            log.debug(
+              `URL ${url} failed to load, the ${
+                this.options.browser
+              } are still on ${startURI} , trying ${
+                tries - index - 1
+              } more time(s) but first wait for ${waitTime} ms.`
+            );
 
+            if (index === tries - 1) {
+              // If the last tries through an error, rethrow as before
+              const message = `Could not load ${url} - the navigation never happend after ${tries} tries and total wait time of ${totalWaitTime} ms`;
+              log.error(message);
+              throw new UrlLoadError(message, url);
+            } else {
+              // We add some wait time before we try again
+              await delay(waitTime);
+              log.info(
+                'Will check again if the browser has navigated to the page'
+              );
+            }
+          } else {
+            // We navigated to a new page, we don't need to test anymore
+            break;
+          }
+        } catch (error) {
+          log.info(
+            `URL failed to load, trying ${tries - index - 1} more time(s): ${
+              error.message
+            }`
+          );
+          //
           if (index === tries - 1) {
             // If the last tries through an error, rethrow as before
-            const message = `Could not load ${url} - the navigation never happend after ${tries} tries and total wait time of ${totalWaitTime} ms`;
-            log.error(message);
-            throw new UrlLoadError(message, url);
+            log.error('Could not load URL %s', url, error);
+            throw new UrlLoadError('Failed to load ' + url, url, {
+              cause: error
+            });
           } else {
-            // We add some wait time before we try again
-            await delay(waitTime);
-            log.info(
-              'Will check again if the browser has navigated to the page'
-            );
+            await delay(1000);
           }
-        } else {
-          // We navigated to a new page, we don't need to test anymore
-          break;
-        }
-      } catch (error) {
-        log.info(
-          `URL failed to load, trying ${tries - index - 1} more time(s): ${
-            error.message
-          }`
-        );
-        //
-        if (index === tries - 1) {
-          // If the last tries through an error, rethrow as before
-          log.error('Could not load URL %s', url, error);
-          throw new UrlLoadError('Failed to load ' + url, url, {
-            cause: error
-          });
-        } else {
-          await delay(1000);
         }
       }
     }

package/lib/firefox/networkManager.js

@@ -0,0 +1,109 @@
+import intel from 'intel';
+import get from 'lodash.get';
+const log = intel.getLogger('browsertime.chrome.network');
+
+export class NetworkManager {
+  constructor(bidi, browsingContextIds, options) {
+    this.bidi = bidi;
+    this.browsingContextIds = browsingContextIds;
+    this.maxTimeout = get(options, 'timeouts.pageCompleteCheck', 30_000);
+    this.idleTime = get(options, 'timeouts.networkIdle', 5000);
+  }
+
+  async waitForNetworkIdle() {
+    await this.bidi.subscribe(
+      'network.beforeRequestSent',
+      this.browsingContextIds
+    );
+    await this.bidi.subscribe(
+      'network.responseCompleted',
+      this.browsingContextIds
+    );
+    await this.bidi.subscribe('network.fetchError', this.browsingContextIds);
+
+    let inflight = 0;
+    let lastRequestTimestamp;
+    let lastResponseTimestamp;
+    this.ws = await this.bidi.socket;
+    this.ws.on('message', function (event) {
+      const { method } = JSON.parse(Buffer.from(event.toString()));
+      if (method) {
+        switch (method) {
+          case 'network.beforeRequestSent': {
+            inflight++;
+            lastRequestTimestamp = Date.now();
+
+            break;
+          }
+          case 'network.responseCompleted': {
+            inflight--;
+            lastResponseTimestamp = Date.now();
+
+            break;
+          }
+          case 'network.fetchError': {
+            inflight--;
+            lastResponseTimestamp = Date.now();
+
+            break;
+          }
+          // No default
+        }
+      }
+    });
+
+    const startTime = Date.now();
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const now = Date.now();
+      const sinceLastResponseRequest =
+        now - Math.max(lastResponseTimestamp, lastRequestTimestamp);
+      const sinceStart = now - startTime;
+
+      if (sinceLastResponseRequest >= this.idleTime) {
+        if (inflight > 0) {
+          log.info(
+            'Idle time without any request/responses. Inflight requests:' +
+              inflight
+          );
+        }
+        await this.bidi.unsubscribe(
+          'network.beforeRequestSent',
+          this.browsingContextIds
+        );
+        await this.bidi.unsubscribe(
+          'network.responseCompleted',
+          this.browsingContextIds
+        );
+
+        await this.bidi.unsubscribe(
+          'network.fetchError',
+          this.browsingContextIds
+        );
+
+        break;
+      }
+
+      if (sinceStart >= this.maxTimeout) {
+        log.info(
+          'Timeout waiting for network idle. Inflight requests:' + inflight
+        );
+        await this.bidi.unsubscribe(
+          'network.beforeRequestSent',
+          this.browsingContextIds
+        );
+        await this.bidi.unsubscribe(
+          'network.responseCompleted',
+          this.browsingContextIds
+        );
+        await this.bidi.unsubscribe(
+          'network.fetchError',
+          this.browsingContextIds
+        );
+        break;
+      }
+
+      await new Promise(r => setTimeout(r, 200));
+    }
+  }
+}

package/lib/firefox/webdriver/builder.js

@@ -175,6 +175,8 @@ export async function configureBuilder(builder, baseDir, options) {
 
   ffOptions.addArguments('-no-remote');
 
+  ffOptions.enableBidi();
+
   // Enable bidi for HAR support
   if (firefoxConfig.bidihar) {
     ffOptions.enableBidi();

package/lib/firefox/webdriver/firefox.js

@@ -12,6 +12,7 @@ import { findFiles } from '../../support/fileUtil.js';
 import { GeckoProfiler } from '../geckoProfiler.js';
 import { MemoryReport } from '../memoryReport.js';
 import { PerfStats } from '../perfStats.js';
+import { NetworkManager } from '../networkManager.js';
 import { getHAR } from '../getHAR.js';
 
 const log = intel.getLogger('browsertime.firefox');
@@ -323,4 +324,11 @@ export class Firefox {
    * Before the browser is stopped/closed.
    */
   async afterBrowserStopped() {}
+
+  async waitForNetworkIdle(driver) {
+    const windowId = await driver.getWindowHandle();
+    const bidi = await driver.getBidi();
+    let network = new NetworkManager(bidi, [windowId], this.options);
+    return network.waitForNetworkIdle();
+  }
 }

package/lib/safari/webdriver/safari.js

@@ -98,4 +98,8 @@ export class Safari {
       });
     }
   }
+
+  async waitForNetworkIdle() {
+    throw new Error('waitForNetworkIdle is not implemented in Safari');
+  }
 }

package/lib/support/cli.js

@@ -176,6 +176,13 @@ export function parseCommandLine() {
         'Timeout when waiting for page to complete loading, in milliseconds',
       group: 'timeouts'
     })
+    .option('timeouts.networkIdle', {
+      default: 5000,
+      type: 'number',
+      describe:
+        'Timeout when running pageCompleteCheckNetworkIdle, in milliseconds',
+      group: 'timeouts'
+    })
     .option('chrome.args', {
       describe:
         'Extra command line arguments to pass to the Chrome process (e.g. --no-sandbox). ' +
@@ -911,6 +918,12 @@ export function parseCommandLine() {
       type: 'boolean',
       default: false
     })
+    .option('pageCompleteCheckNetworkIdle', {
+      describe:
+        'Alternative way to choose when to end your test that works in Chrome and Firefox. Uses CDP or WebDriver Bidi to look at network traffic instead of running JavaScript in the browser to know when to end the test. By default this will wait 5 seconds of inactivity in the network log (no requets/responses in 5 seconds). Use --timeouts.networkIdle to change the 5 seconds. The test will end after 2 minutes if there is still activity on the network. You can change that timout using --timeouts.pageCompleteCheck ',
+      type: 'boolean',
+      default: false
+    })
     .option('pageCompleteCheckPollTimeout', {
       type: 'number',
       default: 1500,

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "browsertime",
   "description": "Get performance metrics from your web page using Browsertime.",
-  "version": "17.17.0",
+  "version": "17.18.1",
   "bin": "./bin/browsertime.js",
   "type": "module",
   "dependencies": {
     "@cypress/xvfb": "1.2.4",
     "@devicefarmer/adbkit": "2.11.3",
-    "@sitespeed.io/chromedriver": "118.0.5993-70",
+    "@sitespeed.io/chromedriver": "119.0.6045-59",
     "@sitespeed.io/edgedriver": "115.0.1901-183",
     "@sitespeed.io/geckodriver": "0.33.0",
     "@sitespeed.io/throttle": "5.0.0",
@@ -29,7 +29,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.12.0",
+    "selenium-webdriver": "4.14.0",
     "yargs": "17.7.2"
   },
   "optionalDependencies": {