browsertime 14.10.0 → 14.12.0

package/CHANGELOG.md

@@ -1,9 +1,35 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 14.12.0 - 2021-11-30
+### Fixed
+* Adding error log message for Chrome/Edge when document request fails (faulty domain etc) [#1682](https://github.com/sitespeedio/browsertime/pull/1682).
+* Added `'devtools.netmonitor.persistlog': true` preference for Firefox to fix the HAR issue introduced in Firefox 94 [#1684](https://github.com/sitespeedio/browsertime/pull/1684).
+
+### Added
+* Updated to Firefox 94 in the Docker image.
+## 14.11.0 - 2021-11-23
+### Fixed
+* Use the viewport to determine if more cropping is needed in visual metrics. Thank you [Gregory Mierzwinski](https://github.com/gmierz) for PR [#1680](https://github.com/sitespeedio/browsertime/pull/1680).
+### Added
+* Updated to Selenium 4.1.0 [#1679](https://github.com/sitespeedio/browsertime/pull/1679) 
+
+## 14.10.2 - 2021-11-20
+### Fixed
+* Disabled the version check for Edge/Edgedriver in Edgedriver [#1678](https://github.com/sitespeedio/browsertime/pull/1678).
+
+## 14.10.1 - 2021-11-19
+### Fixed
+* Disabled the automatic Chrome/Chromedriver version check in Chromedriver [#1676](https://github.com/sitespeedio/browsertime/pull/1676).
+* Loop until we find a frame with a good viewport or until we run out of retries for Visual Metrics. Thank you [Gregory Mierzwinski](https://github.com/gmierz) for PR [#1668](https://github.com/sitespeedio/browsertime/pull/1668). 
+
 ## 14.10.0 - 2021-11-16
 ### Added
 * Updated to Chromedriver 96 and Chrome 96 in the Docker container [#1670](https://github.com/sitespeedio/browsertime/pull/1670).
 
+### Fixed
+* Added checks for Firefox HAR that it actually has a page.
+* Updated day js.
+
 ## 14.9.0 - 2021-11-07
 ### Added
 * Updated to new Chome HAR PR [#1666](https://github.com/sitespeedio/browsertime/pull/1666) that inlcudes chunk information.

package/browsertime/visualmetrics.py

@@ -44,12 +44,12 @@ import shutil
 import subprocess
 import tempfile
 
-if (sys.version_info > (3, 0)):
-    GZIP_TEXT = 'wt'
-    GZIP_READ_TEXT = 'rt'
+if sys.version_info > (3, 0):
+    GZIP_TEXT = "wt"
+    GZIP_READ_TEXT = "rt"
 else:
-    GZIP_TEXT = 'w'
-    GZIP_READ_TEXT = 'r'
+    GZIP_TEXT = "w"
+    GZIP_READ_TEXT = "r"
 
 # Globals
 options = None
@@ -72,6 +72,9 @@ def video_to_frames(
     multiple,
     find_viewport,
     viewport_time,
+    viewport_retries,
+    viewport_min_height,
+    viewport_min_width,
     full_resolution,
     timeline_file,
     trim_end,
@@ -92,8 +95,14 @@ def video_to_frames(
                 os.mkdir(directory, 0o755)
             if os.path.isdir(directory):
                 directory = os.path.realpath(directory)
-                viewport = find_video_viewport(
-                    video, directory, find_viewport, viewport_time
+                viewport, cropped = find_video_viewport(
+                    video,
+                    directory,
+                    find_viewport,
+                    viewport_time,
+                    viewport_retries,
+                    viewport_min_height,
+                    viewport_min_width,
                 )
                 gc.collect()
                 if extract_frames(video, directory, full_resolution, viewport):
@@ -113,12 +122,12 @@ def video_to_frames(
                             remove_orange_frames(dir, orange_file)
                         find_first_frame(dir, white_file)
                         blank_first_frame(dir)
-                        find_render_start(dir, orange_file, gray_file)
+                        find_render_start(dir, orange_file, gray_file, cropped)
                         find_last_frame(dir, white_file)
                         adjust_frame_times(dir)
                         if timeline_file is not None and not multiple:
                             synchronize_to_timeline(dir, timeline_file)
-                        eliminate_duplicate_frames(dir)
+                        eliminate_duplicate_frames(dir, cropped)
                         eliminate_similar_frames(dir)
                         # See if we are limiting the number of frames to keep
                         # (before processing them to save processing time)
@@ -169,10 +178,10 @@ def extract_frames(video, directory, full_resolution, viewport):
         ]
         logging.debug(" ".join(command))
         lines = []
-        if (sys.version_info > (3, 0)):
-             proc = subprocess.Popen(command, stderr=subprocess.PIPE, encoding='UTF-8')
+        if sys.version_info > (3, 0):
+            proc = subprocess.Popen(command, stderr=subprocess.PIPE, encoding="UTF-8")
         else:
-             proc = subprocess.Popen(command, stderr=subprocess.PIPE)
+            proc = subprocess.Popen(command, stderr=subprocess.PIPE)
         while proc.poll() is None:
             lines.extend(iter(proc.stderr.readline, ""))
 
@@ -354,68 +363,124 @@ def find_image_viewport(file):
     return viewport
 
 
-def find_video_viewport(video, directory, find_viewport, viewport_time):
+def find_video_viewport(
+    video,
+    directory,
+    find_viewport,
+    viewport_time,
+    viewport_retries,
+    viewport_min_height,
+    viewport_min_width,
+):
     logging.debug("Finding Video Viewport...")
     viewport = None
+
+    # cropped will be True if the viewport setting changes
+    # the original frame
+    cropped = False
+
     try:
         from PIL import Image
 
-        frame = os.path.join(directory, "viewport.png")
-        if os.path.isfile(frame):
-            os.remove(frame)
-        command = ["ffmpeg", "-i", video]
-        if viewport_time:
-            command.extend(["-ss", viewport_time])
-        command.extend(["-frames:v", "1", frame])
-        subprocess.check_output(command)
-        if os.path.isfile(frame):
-            with Image.open(frame) as im:
-                width, height = im.size
-                logging.debug("%s is %dx%d", frame, width, height)
-            if options.notification:
-                im = Image.open(frame)
-                pixels = im.load()
-                middle = int(math.floor(height / 2))
-                # Find the top edge (at ~40% in to deal with browsers that
-                # color the notification area)
-                x = int(width * 0.4)
-                y = 0
-                background = pixels[x, y]
-                top = None
-                while top is None and y < middle:
-                    if not colors_are_similar(background, pixels[x, y]):
-                        top = y
-                    else:
-                        y += 1
-                if top is None:
-                    top = 0
-                logging.debug("Window top edge is {0:d}".format(top))
-
-                # Find the bottom edge
-                x = 0
-                y = height - 1
-                bottom = None
-                while bottom is None and y > middle:
-                    if not colors_are_similar(background, pixels[x, y]):
-                        bottom = y
-                    else:
-                        y -= 1
-                if bottom is None:
-                    bottom = height - 1
-                logging.debug("Window bottom edge is {0:d}".format(bottom))
+        retries = -1
+
+        while (
+            viewport is None
+            or viewport["height"] <= viewport_min_height
+            or viewport["width"] <= viewport_min_width
+        ):
+            retries += 1
+            if retries >= 1:
+                # In some cases, the first frame is not an orange screen or a screen
+                # with a solid color. In this case, we need to try finding the viewport
+                # using the next frame. The `viewport_retries` dictates the maximum number
+                # of frames to check.
+                if retries >= viewport_retries:
+                    logging.exception(
+                        "Could not calculate a viewport after %s tries.",
+                        viewport_retries,
+                    )
+                    break
+                logging.info("Failed to find a good viewport. Retrying...")
 
-                viewport = {"x": 0, "y": top, "width": width, "height": (bottom - top)}
+            logging.debug("Using frame " + str(retries))
 
-            elif find_viewport:
-                viewport = find_image_viewport(frame)
-            else:
-                viewport = {"x": 0, "y": 0, "width": width, "height": height}
-            os.remove(frame)
+            frame = os.path.join(directory, "viewport.png")
+            if os.path.isfile(frame):
+                os.remove(frame)
 
-    except Exception:
+            command = ["ffmpeg", "-i", video]
+            if viewport_time:
+                command.extend(["-ss", viewport_time])
+
+            # Pull one frame from the video starting with the frame at
+            # the `retries` index
+            command.extend(["-vf", "select=gte(n\\,%s)" % retries])
+            command.extend(["-frames:v", "1", frame])
+            subprocess.check_output(command)
+
+            if os.path.isfile(frame):
+                with Image.open(frame) as im:
+                    width, height = im.size
+                    logging.debug("%s is %dx%d", frame, width, height)
+                if options.notification:
+                    im = Image.open(frame)
+                    pixels = im.load()
+                    middle = int(math.floor(height / 2))
+                    # Find the top edge (at ~40% in to deal with browsers that
+                    # color the notification area)
+                    x = int(width * 0.4)
+                    y = 0
+                    background = pixels[x, y]
+                    top = None
+                    while top is None and y < middle:
+                        if not colors_are_similar(background, pixels[x, y]):
+                            top = y
+                        else:
+                            y += 1
+                    if top is None:
+                        top = 0
+                    logging.debug("Window top edge is {0:d}".format(top))
+
+                    # Find the bottom edge
+                    x = 0
+                    y = height - 1
+                    bottom = None
+                    while bottom is None and y > middle:
+                        if not colors_are_similar(background, pixels[x, y]):
+                            bottom = y
+                        else:
+                            y -= 1
+                    if bottom is None:
+                        bottom = height - 1
+                    logging.debug("Window bottom edge is {0:d}".format(bottom))
+
+                    viewport = {
+                        "x": 0,
+                        "y": top,
+                        "width": width,
+                        "height": (bottom - top),
+                    }
+
+                elif find_viewport:
+                    viewport = find_image_viewport(frame)
+                else:
+                    viewport = {"x": 0, "y": 0, "width": width, "height": height}
+
+                os.remove(frame)
+
+        if viewport is not None and viewport != {
+            "x": 0,
+            "y": 0,
+            "width": width,
+            "height": height,
+        }:
+            cropped = True
+
+    except Exception as e:
         viewport = None
 
-    return viewport
+    return viewport, cropped
 
 
 def trim_video_end(directory, trim_time):
@@ -564,7 +629,7 @@ def find_last_frame(directory, white_file):
         logging.exception("Error finding last frame")
 
 
-def find_render_start(directory, orange_file, gray_file):
+def find_render_start(directory, orange_file, gray_file, cropped):
     logging.debug("Finding Render Start...")
     try:
         if (
@@ -590,6 +655,10 @@ def find_render_start(directory, orange_file, gray_file):
                     mask["y"] = int(math.floor(height / 2 - mask["height"] / 2))
                 else:
                     mask = None
+
+                im_width = width
+                im_height = height
+
                 top = 10
                 right_margin = 10
                 bottom_margin = 24
@@ -609,6 +678,14 @@ def find_render_start(directory, orange_file, gray_file):
                     width = max(client_viewport["width"] - right_margin, 1)
                     left += client_viewport["x"]
                     top += client_viewport["y"]
+                elif cropped:
+                    # The image was already cropped, so only cutout the bottom
+                    # to get rid of the network request/etc. information
+                    top = 0
+                    left = 0
+                    width = im_width
+                    height = im_height - bottom_margin
+
                 crop = "{0:d}x{1:d}+{2:d}+{3:d}".format(width, height, left, top)
                 for i in range(1, count):
                     if frames_match(first, files[i], 10, 0, crop, mask):
@@ -628,7 +705,7 @@ def find_render_start(directory, orange_file, gray_file):
         logging.exception("Error getting render start")
 
 
-def eliminate_duplicate_frames(directory):
+def eliminate_duplicate_frames(directory, cropped):
     logging.debug("Eliminating Duplicate Frames...")
     global client_viewport
     try:
@@ -646,6 +723,9 @@ def eliminate_duplicate_frames(directory):
                 ):
                     client_viewport = None
 
+            im_width = width
+            im_height = height
+
             # Figure out the region of the image that we care about
             top = 40
             right_margin = 10
@@ -663,6 +743,13 @@ def eliminate_duplicate_frames(directory):
                 width = max(client_viewport["width"] - right_margin, 1)
                 left += client_viewport["x"]
                 top += client_viewport["y"]
+            elif cropped:
+                # The image was already cropped, so only cutout the bottom
+                # to get rid of the network request/etc. information
+                top = 0
+                left = 0
+                width = im_width
+                height = im_height - bottom_margin
 
             crop = "{0:d}x{1:d}+{2:d}+{3:d}".format(width, height, left, top)
             logging.debug("Viewport cropping set to " + crop)
@@ -787,10 +874,14 @@ def crop_viewport(directory):
 def get_decimate_filter():
     decimate = None
     try:
-        if (sys.version_info > (3, 0)):
-            filters = subprocess.check_output(['ffmpeg', '-filters'], stderr=subprocess.STDOUT, encoding='UTF-8')
+        if sys.version_info > (3, 0):
+            filters = subprocess.check_output(
+                ["ffmpeg", "-filters"], stderr=subprocess.STDOUT, encoding="UTF-8"
+            )
         else:
-            filters = subprocess.check_output(['ffmpeg', '-filters'], stderr=subprocess.STDOUT)
+            filters = subprocess.check_output(
+                ["ffmpeg", "-filters"], stderr=subprocess.STDOUT
+            )
         lines = filters.split("\n")
         match = re.compile(
             r"(?P<filter>[\w]*decimate).*V->V.*Remove near-duplicate frames"
@@ -864,10 +955,14 @@ def is_color_frame(file, color_file):
                     image_magick["compare"],
                 )
 
-                if (sys.version_info > (3, 0)):
-                   compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True, encoding='UTF-8')
+                if sys.version_info > (3, 0):
+                    compare = subprocess.Popen(
+                        command, stderr=subprocess.PIPE, shell=True, encoding="UTF-8"
+                    )
                 else:
-                    compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True)
+                    compare = subprocess.Popen(
+                        command, stderr=subprocess.PIPE, shell=True
+                    )
                 out, err = compare.communicate()
                 if re.match("^[0-9]+$", err):
                     different_pixels = int(err)
@@ -908,9 +1003,11 @@ def is_white_frame(file, white_file):
             ).format(
                 image_magick["convert"], white_file, file, crop, image_magick["compare"]
             )
-        if (sys.version_info > (3, 0)):
-            compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True, encoding='UTF-8')
-        else:    
+        if sys.version_info > (3, 0):
+            compare = subprocess.Popen(
+                command, stderr=subprocess.PIPE, shell=True, encoding="UTF-8"
+            )
+        else:
             compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True)
         out, err = compare.communicate()
         if re.match("^[0-9]+$", err):
@@ -966,9 +1063,11 @@ def frames_match(image1, image2, fuzz_percent, max_differences, crop_region, mas
     )
     if platform.system() != "Windows":
         command = command.replace("(", "\\(").replace(")", "\\)")
-    if (sys.version_info > (3, 0)):
-        compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True, encoding='UTF-8')
-    else:    
+    if sys.version_info > (3, 0):
+        compare = subprocess.Popen(
+            command, stderr=subprocess.PIPE, shell=True, encoding="UTF-8"
+        )
+    else:
         compare = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True)
     out, err = compare.communicate()
     if re.match("^[0-9]+$", err):
@@ -1686,7 +1785,7 @@ def calculate_contentful_speed_index(progress, directory):
             command = "{0} {1} -canny 2x2+8%+8% -define histogram:unique-colors=true -format %c histogram:info:-".format(
                 image_magick["convert"], current_frame
             )
-            output = subprocess.check_output(command, shell=True).decode('utf-8')
+            output = subprocess.check_output(command, shell=True).decode("utf-8")
             logging.debug("Output %s" % output)
 
             # Take the last matching pixel count, which is the pixel count from the last
@@ -1887,28 +1986,28 @@ def calculate_hero_time(progress, directory, hero, viewport):
 def check_config():
     ok = True
 
-    print("ffmpeg:  ",)
+    print("ffmpeg:  ")
     if get_decimate_filter() is not None:
         print("OK")
     else:
         print("FAIL")
         ok = False
 
-    print("convert: ",)
+    print("convert: ")
     if check_process("{0} -version".format(image_magick["convert"]), "ImageMagick"):
         print("OK")
     else:
         print("FAIL")
         ok = False
 
-    print("compare: ",)
+    print("compare: ")
     if check_process("{0} -version".format(image_magick["compare"]), "ImageMagick"):
         print("OK")
     else:
         print("FAIL")
         ok = False
 
-    print("Pillow:  ",)
+    print("Pillow:  ")
     try:
         from PIL import Image, ImageDraw  # noqa
 
@@ -1917,7 +2016,7 @@ def check_config():
         print("FAIL")
         ok = False
 
-    print("SSIM:    ",)
+    print("SSIM:    ")
     try:
         from ssim import compute_ssim  # noqa
 
@@ -1932,8 +2031,10 @@ def check_config():
 def check_process(command, output):
     ok = False
     try:
-        if (sys.version_info > (3, 0)):
-            out = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True, encoding='UTF-8')
+        if sys.version_info > (3, 0):
+            out = subprocess.check_output(
+                command, stderr=subprocess.STDOUT, shell=True, encoding="UTF-8"
+            )
         else:
             out = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
         if out.find(output) > -1:
@@ -1976,7 +2077,7 @@ def main():
         "--logfile", help="Write log messages to given file instead of stdout"
     )
     parser.add_argument(
-        '--logformat',
+        "--logformat",
         help="Formatting for the log messages",
         default="%(asctime)s.%(msecs)03d - %(message)s",
     )
@@ -2081,6 +2182,28 @@ def main():
         help="Time of the video frame to use for identifying the viewport "
         "(in HH:MM:SS.xx format).",
     )
+    parser.add_argument(
+        "--viewportretries",
+        type=int,
+        default=5,
+        help="Number of times to attempt to obtain a viewport. Analagous to the "
+        "number of frames to try to find a viewport with. By default, up to the "
+        "first 5 frames are used.",
+    )
+    parser.add_argument(
+        "--viewportminheight",
+        type=int,
+        default=0,
+        help="The minimum possible height (in pixels) for the viewport. Used when "
+        "attempting to find the viewport size. Defaults to 0.",
+    )
+    parser.add_argument(
+        "--viewportminwidth",
+        type=int,
+        default=0,
+        help="The minimum possible width (in pixels) for the viewport. Used when "
+        "attempting to find the viewport size. Defaults to 0.",
+    )
     parser.add_argument(
         "-s",
         "--start",
@@ -2213,9 +2336,7 @@ def main():
         )
     else:
         logging.basicConfig(
-            level=log_level,
-            format=options.logformat,
-            datefmt="%H:%M:%S",
+            level=log_level, format=options.logformat, datefmt="%H:%M:%S"
         )
 
     if options.multiple:
@@ -2285,6 +2406,9 @@ def main():
                     options.multiple,
                     options.viewport,
                     options.viewporttime,
+                    options.viewportretries,
+                    options.viewportminheight,
+                    options.viewportminwidth,
                     options.full,
                     options.timeline,
                     options.trimend,

package/lib/chrome/chromeDevtoolsProtocol.js

@@ -161,6 +161,14 @@ class ChromeDevtoolsProtocol {
     return this.cdpClient.Network.clearBrowserCookies();
   }
 
+  async loadingFailed() {
+    return this.cdpClient.Network.loadingFailed(param => {
+      if (param.type === 'Document') {
+        log.error('Could not load document:' + param.errorText);
+      }
+    });
+  }
+
   async setRequestHeaders(requestHeaders) {
     // Our cli don't validate parameters
     // so -run will become -r etc

package/lib/chrome/webdriver/builder.js

@@ -25,6 +25,10 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
     }
 
     const serviceBuilder = new chrome.ServiceBuilder(chromedriverPath);
+
+    // Remove the check that matches the Chromedriver version with Chrome version.
+    serviceBuilder.addArguments('--disable-build-check');
+
     if (options.chrome && options.chrome.chromedriverPort) {
       serviceBuilder.setPort(options.chrome.chromedriverPort);
     }
@@ -38,7 +42,6 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
         serviceBuilder.enableVerboseLogging();
     }
     chrome.setDefaultService(serviceBuilder.build());
-
     hasConfiguredChromeDriverService = true;
   }
 

package/lib/chrome/webdriver/chromium.js

@@ -107,6 +107,8 @@ class Chromium {
 
     await this.cdpClient.setupLongTask();
 
+    await this.cdpClient.loadingFailed();
+
     // Make sure we clear the console log
     // Hopefully one time is enough?
     return runner.getLogs(Type.BROWSER);

package/lib/edge/webdriver/builder.js

@@ -25,6 +25,10 @@ module.exports.configureBuilder = function (builder, baseDir, options) {
   }
 
   const serviceBuilder = new edge.ServiceBuilder(edgedriverPath);
+
+  // Remove the check that matches the Edgedriver version with Edge version.
+  serviceBuilder.addArguments('--disable-build-check');
+
   if (
     options.verbose >= 2 ||
     chromeConfig.enableChromeDriverLog ||

package/lib/firefox/settings/firefoxPreferences.js

@@ -179,5 +179,8 @@ module.exports = {
 
   // disable calls to detectportal.firefox.com
   'network.captive-portal-service.enabled': false,
-  'network.connectivity-service.enabled': false
+  'network.connectivity-service.enabled': false,
+  // https://bugzilla.mozilla.org/show_bug.cgi?id=1740116#c15
+  // https://github.com/sitespeedio/browsertime/issues/1671
+  'devtools.netmonitor.persistlog': true
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "description": "Browsertime",
-  "version": "14.10.0",
+  "version": "14.12.0",
   "bin": "./bin/browsertime.js",
   "dependencies": {
     "@sitespeed.io/throttle": "3.0.0",
@@ -22,7 +22,7 @@
     "lodash.merge": "4.6.2",
     "lodash.pick": "4.4.0",
     "lodash.set": "4.3.2",
-    "selenium-webdriver": "4.0.0",
+    "selenium-webdriver": "4.1.0",
     "@sitespeed.io/edgedriver": "95.0.1020-30",
     "@sitespeed.io/geckodriver": "0.29.1-2",
     "@sitespeed.io/tracium": "0.3.3",