browsertime 14.10.1 → 14.12.1

package/CHANGELOG.md

@@ -1,8 +1,29 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 14.12.1 - 2021-12-01
+### Fixed
+* Changed log level to debug for Chrome/Edge when document request fails. It turns out that IRL that happens a lot and spam the log. Lets iterate over that functionality.
+## 14.12.0 - 2021-11-30
+### Fixed
+* Adding error log message for Chrome/Edge when document request fails (faulty domain etc) [#1682](https://github.com/sitespeedio/browsertime/pull/1682).
+* Added `'devtools.netmonitor.persistlog': true` preference for Firefox to fix the HAR issue introduced in Firefox 94 [#1684](https://github.com/sitespeedio/browsertime/pull/1684).
+
+### Added
+* Updated to Firefox 94 in the Docker image.
+## 14.11.0 - 2021-11-23
+### Fixed
+* Use the viewport to determine if more cropping is needed in visual metrics. Thank you [Gregory Mierzwinski](https://github.com/gmierz) for PR [#1680](https://github.com/sitespeedio/browsertime/pull/1680).
+### Added
+* Updated to Selenium 4.1.0 [#1679](https://github.com/sitespeedio/browsertime/pull/1679) 
+
+## 14.10.2 - 2021-11-20
+### Fixed
+* Disabled the version check for Edge/Edgedriver in Edgedriver [#1678](https://github.com/sitespeedio/browsertime/pull/1678).
+
 ## 14.10.1 - 2021-11-19
 ### Fixed
 * Disabled the automatic Chrome/Chromedriver version check in Chromedriver [#1676](https://github.com/sitespeedio/browsertime/pull/1676).
+* Loop until we find a frame with a good viewport or until we run out of retries for Visual Metrics. Thank you [Gregory Mierzwinski](https://github.com/gmierz) for PR [#1668](https://github.com/sitespeedio/browsertime/pull/1668). 
 
 ## 14.10.0 - 2021-11-16
 ### Added

package/browsertime/visualmetrics.py

@@ -95,7 +95,7 @@ def video_to_frames(
                 os.mkdir(directory, 0o755)
             if os.path.isdir(directory):
                 directory = os.path.realpath(directory)
-                viewport = find_video_viewport(
+                viewport, cropped = find_video_viewport(
                     video,
                     directory,
                     find_viewport,
@@ -122,12 +122,12 @@ def video_to_frames(
                             remove_orange_frames(dir, orange_file)
                         find_first_frame(dir, white_file)
                         blank_first_frame(dir)
-                        find_render_start(dir, orange_file, gray_file)
+                        find_render_start(dir, orange_file, gray_file, cropped)
                         find_last_frame(dir, white_file)
                         adjust_frame_times(dir)
                         if timeline_file is not None and not multiple:
                             synchronize_to_timeline(dir, timeline_file)
-                        eliminate_duplicate_frames(dir)
+                        eliminate_duplicate_frames(dir, cropped)
                         eliminate_similar_frames(dir)
                         # See if we are limiting the number of frames to keep
                         # (before processing them to save processing time)
@@ -374,6 +374,11 @@ def find_video_viewport(
 ):
     logging.debug("Finding Video Viewport...")
     viewport = None
+
+    # cropped will be True if the viewport setting changes
+    # the original frame
+    cropped = False
+
     try:
         from PIL import Image
 
@@ -461,12 +466,21 @@ def find_video_viewport(
                     viewport = find_image_viewport(frame)
                 else:
                     viewport = {"x": 0, "y": 0, "width": width, "height": height}
+
                 os.remove(frame)
 
+        if viewport is not None and viewport != {
+            "x": 0,
+            "y": 0,
+            "width": width,
+            "height": height,
+        }:
+            cropped = True
+
     except Exception as e:
         viewport = None
 
-    return viewport
+    return viewport, cropped
 
 
 def trim_video_end(directory, trim_time):
@@ -615,7 +629,7 @@ def find_last_frame(directory, white_file):
         logging.exception("Error finding last frame")
 
 
-def find_render_start(directory, orange_file, gray_file):
+def find_render_start(directory, orange_file, gray_file, cropped):
     logging.debug("Finding Render Start...")
     try:
         if (
@@ -641,6 +655,10 @@ def find_render_start(directory, orange_file, gray_file):
                     mask["y"] = int(math.floor(height / 2 - mask["height"] / 2))
                 else:
                     mask = None
+
+                im_width = width
+                im_height = height
+
                 top = 10
                 right_margin = 10
                 bottom_margin = 24
@@ -660,6 +678,14 @@ def find_render_start(directory, orange_file, gray_file):
                     width = max(client_viewport["width"] - right_margin, 1)
                     left += client_viewport["x"]
                     top += client_viewport["y"]
+                elif cropped:
+                    # The image was already cropped, so only cutout the bottom
+                    # to get rid of the network request/etc. information
+                    top = 0
+                    left = 0
+                    width = im_width
+                    height = im_height - bottom_margin
+
                 crop = "{0:d}x{1:d}+{2:d}+{3:d}".format(width, height, left, top)
                 for i in range(1, count):
                     if frames_match(first, files[i], 10, 0, crop, mask):
@@ -679,7 +705,7 @@ def find_render_start(directory, orange_file, gray_file):
         logging.exception("Error getting render start")
 
 
-def eliminate_duplicate_frames(directory):
+def eliminate_duplicate_frames(directory, cropped):
     logging.debug("Eliminating Duplicate Frames...")
     global client_viewport
     try:
@@ -697,6 +723,9 @@ def eliminate_duplicate_frames(directory):
                 ):
                     client_viewport = None
 
+            im_width = width
+            im_height = height
+
             # Figure out the region of the image that we care about
             top = 40
             right_margin = 10
@@ -714,6 +743,13 @@ def eliminate_duplicate_frames(directory):
                 width = max(client_viewport["width"] - right_margin, 1)
                 left += client_viewport["x"]
                 top += client_viewport["y"]
+            elif cropped:
+                # The image was already cropped, so only cutout the bottom
+                # to get rid of the network request/etc. information
+                top = 0
+                left = 0
+                width = im_width
+                height = im_height - bottom_margin
 
             crop = "{0:d}x{1:d}+{2:d}+{3:d}".format(width, height, left, top)
             logging.debug("Viewport cropping set to " + crop)

package/lib/chrome/chromeDevtoolsProtocol.js

@@ -161,6 +161,14 @@ class ChromeDevtoolsProtocol {
     return this.cdpClient.Network.clearBrowserCookies();
   }
 
+  async loadingFailed() {
+    return this.cdpClient.Network.loadingFailed(param => {
+      if (param.type === 'Document') {
+        log.debug('Could not load document:' + param.errorText);
+      }
+    });
+  }
+
   async setRequestHeaders(requestHeaders) {
     // Our cli don't validate parameters
     // so -run will become -r etc

package/lib/chrome/webdriver/chromium.js

@@ -107,6 +107,8 @@ class Chromium {
 
     await this.cdpClient.setupLongTask();
 
+    await this.cdpClient.loadingFailed();
+
     // Make sure we clear the console log
     // Hopefully one time is enough?
     return runner.getLogs(Type.BROWSER);

package/lib/edge/webdriver/builder.js

@@ -25,6 +25,10 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
   }
 
   const serviceBuilder = new edge.ServiceBuilder(edgedriverPath);
+
+  // Remove the check that matches the Edgedriver version with Edge version.
+  serviceBuilder.addArguments('--disable-build-check');
+
   if (
     options.verbose >= 2 ||
     chromeConfig.enableChromeDriverLog ||

package/lib/firefox/settings/firefoxPreferences.js

@@ -179,5 +179,8 @@ module.exports = {
 
   // disable calls to detectportal.firefox.com
   'network.captive-portal-service.enabled': false,
-  'network.connectivity-service.enabled': false
+  'network.connectivity-service.enabled': false,
+  // https://bugzilla.mozilla.org/show_bug.cgi?id=1740116#c15
+  // https://github.com/sitespeedio/browsertime/issues/1671
+  'devtools.netmonitor.persistlog': true
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "description": "Browsertime",
-  "version": "14.10.1",
+  "version": "14.12.1",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@sitespeed.io/throttle": "3.0.0",
@@ -22,7 +22,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.0.0",
+    "selenium-webdriver": "4.1.0",
     "@sitespeed.io/edgedriver": "95.0.1020-30",
     "@sitespeed.io/geckodriver": "0.29.1-2",
     "@sitespeed.io/tracium": "0.3.3",