ferrum 0.13 → 0.15

data/lib/ferrum/browser/command.rb

@@ -39,10 +39,18 @@ module Ferrum
         !!options.xvfb
       end
 
+      def headless_new?
+        @flags["headless"] == "new"
+      end
+
       def to_a
         [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
       end
 
+      def to_s
+        to_a.join(" \\ \n  ")
+      end
+
       private
 
       def merge_options

data/lib/ferrum/browser/options/chrome.rb

@@ -6,7 +6,6 @@ module Ferrum
       class Chrome < Base
         DEFAULT_OPTIONS = {
           "headless" => nil,
-          "disable-gpu" => nil,
           "hide-scrollbars" => nil,
           "mute-audio" => nil,
           "enable-automation" => nil,
@@ -19,8 +18,9 @@ module Ferrum
           "keep-alive-for-test" => nil,
           "disable-popup-blocking" => nil,
           "disable-extensions" => nil,
+          "disable-component-extensions-with-background-pages" => nil,
           "disable-hang-monitor" => nil,
-          "disable-features" => "site-per-process,TranslateUI",
+          "disable-features" => "site-per-process,IsolateOrigins,TranslateUI",
           "disable-translate" => nil,
           "disable-background-networking" => nil,
           "enable-features" => "NetworkService,NetworkServiceInProcess",
@@ -32,11 +32,13 @@ module Ferrum
           "disable-ipc-flooding-protection" => nil,
           "disable-prompt-on-repost" => nil,
           "disable-renderer-backgrounding" => nil,
+          "disable-site-isolation-trials" => nil,
           "force-color-profile" => "srgb",
           "metrics-recording-only" => nil,
           "safebrowsing-disable-auto-update" => nil,
           "password-store" => "basic",
-          "no-startup-window" => nil
+          "no-startup-window" => nil,
+          "remote-allow-origins" => "*"
           # NOTE: --no-sandbox is not needed if you properly setup a user in the container.
           # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
           # "no-sandbox" => nil,
@@ -61,7 +63,6 @@ module Ferrum
         def merge_required(flags, options, user_data_dir)
           flags = flags.merge("remote-debugging-port" => options.port,
                               "remote-debugging-address" => options.host,
-                              # Doesn't work on MacOS, so we need to set it by CDP
                               "window-size" => options.window_size&.join(","),
                               "user-data-dir" => user_data_dir)
 
@@ -74,9 +75,20 @@ module Ferrum
         end
 
         def merge_default(flags, options)
-          defaults = except("headless", "disable-gpu") unless options.headless
+          defaults = case options.headless
+                     when false
+                       except("headless", "disable-gpu")
+                     when "new"
+                       except("headless").merge("headless" => "new")
+                     end
 
           defaults ||= DEFAULT_OPTIONS
+          # On Windows, the --disable-gpu flag is a temporary work around for a few bugs.
+          # See https://bugs.chromium.org/p/chromium/issues/detail?id=737678 for more information.
+          defaults = defaults.merge("disable-gpu" => nil) if Utils::Platform.windows?
+          # Use Metal on Apple Silicon
+          # https://github.com/google/angle#platform-support-via-backing-renderers
+          defaults = defaults.merge("use-angle" => "metal") if Utils::Platform.mac_arm?
           defaults.merge(flags)
         end
       end

data/lib/ferrum/browser/options.rb

@@ -3,7 +3,6 @@
 module Ferrum
   class Browser
     class Options
-      HEADLESS = true
       BROWSER_PORT = "0"
       BROWSER_HOST = "127.0.0.1"
       WINDOW_SIZE = [1024, 768].freeze
@@ -12,55 +11,56 @@ module Ferrum
       PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
       DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
 
-      attr_reader :window_size, :timeout, :logger, :ws_max_receive_size,
+      attr_reader :window_size, :logger, :ws_max_receive_size,
                   :js_errors, :base_url, :slowmo, :pending_connection_errors,
-                  :url, :env, :process_timeout, :browser_name, :browser_path,
-                  :save_path, :extensions, :proxy, :port, :host, :headless,
-                  :ignore_default_browser_options, :browser_options, :xvfb
+                  :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
+                  :save_path, :proxy, :port, :host, :headless, :browser_options,
+                  :ignore_default_browser_options, :xvfb, :flatten
+      attr_accessor :timeout, :default_user_agent
 
       def initialize(options = nil)
         @options = Hash(options&.dup)
+
         @port = @options.fetch(:port, BROWSER_PORT)
         @host = @options.fetch(:host, BROWSER_HOST)
         @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
         @window_size = @options.fetch(:window_size, WINDOW_SIZE)
         @js_errors = @options.fetch(:js_errors, false)
-        @headless = @options.fetch(:headless, HEADLESS)
+        @headless = @options.fetch(:headless, true)
+        @flatten = @options.fetch(:flatten, true)
         @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
-        @browser_options = @options.fetch(:browser_options, {})
         @slowmo = @options[:slowmo].to_f
 
-        @ws_max_receive_size, @env, @browser_name, @browser_path,
-          @save_path, @extensions, @ignore_default_browser_options, @xvfb = @options.values_at(
-            :ws_max_receive_size, :env, :browser_name, :browser_path, :save_path, :extensions,
-            :ignore_default_browser_options, :xvfb
-          )
+        @env = @options[:env]
+        @xvfb = @options[:xvfb]
+        @save_path = @options[:save_path]
+        @browser_name = @options[:browser_name]
+        @browser_path = @options[:browser_path]
+        @ws_max_receive_size = @options[:ws_max_receive_size]
+        @ignore_default_browser_options = @options[:ignore_default_browser_options]
 
-        @options[:window_size] = @window_size
-        @proxy = parse_proxy(@options[:proxy])
+        @proxy = validate_proxy(@options[:proxy])
         @logger = parse_logger(@options[:logger])
         @base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
         @url = @options[:url].to_s if @options[:url]
+        @ws_url = @options[:ws_url].to_s if @options[:ws_url]
 
-        @options.freeze
-        @browser_options.freeze
+        @options = @options.merge(window_size: @window_size).freeze
+        @browser_options = @options.fetch(:browser_options, {}).freeze
       end
 
-      def to_h
-        @options
+      def base_url=(value)
+        @base_url = parse_base_url(value)
       end
 
-      def parse_base_url(value)
-        parsed = Addressable::URI.parse(value)
-        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
-          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+      def extensions
+        @extensions ||= Array(@options[:extensions]).map do |extension|
+          (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
         end
-
-        parsed
       end
 
-      def parse_proxy(options)
+      def validate_proxy(options)
         return unless options
 
         raise ArgumentError, "proxy options must be a Hash" unless options.is_a?(Hash)
@@ -72,6 +72,10 @@ module Ferrum
         options
       end
 
+      def to_h
+        @options
+      end
+
       private
 
       def parse_logger(logger)
@@ -79,6 +83,15 @@ module Ferrum
 
         !logger && DEBUG_MODE ? $stdout.tap { |s| s.sync = true } : logger
       end
+
+      def parse_base_url(value)
+        parsed = Addressable::URI.parse(value)
+        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
+          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+        end
+
+        parsed
+      end
     end
   end
 end

data/lib/ferrum/browser/process.rb

@@ -62,11 +62,15 @@ module Ferrum
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
 
+        if options.ws_url
+          response = parse_json_version(options.ws_url)
+          self.ws_url = response&.[]("webSocketDebuggerUrl") || options.ws_url
+          return
+        end
+
         if options.url
-          url = URI.join(options.url, "/json/version")
-          response = JSON.parse(::Net::HTTP.get(url))
-          self.ws_url = response["webSocketDebuggerUrl"]
-          parse_browser_versions
+          response = parse_json_version(options.url)
+          self.ws_url = response&.[]("webSocketDebuggerUrl")
           return
         end
 
@@ -100,7 +104,7 @@ module Ferrum
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
-          parse_browser_versions
+          parse_json_version(ws_url)
         ensure
           close_io(read_io, write_io)
         end
@@ -122,6 +126,17 @@ module Ferrum
         start
       end
 
+      def inspect
+        "#<#{self.class} " \
+          "@user_data_dir=#{@user_data_dir.inspect} " \
+          "@command=#<#{@command.class}:#{@command.object_id}> " \
+          "@default_user_agent=#{@default_user_agent.inspect} " \
+          "@ws_url=#{@ws_url.inspect} " \
+          "@v8_version=#{@v8_version.inspect} " \
+          "@browser_version=#{@browser_version.inspect} " \
+          "@webkit_version=#{@webkit_version.inspect}>"
+      end
+
       private
 
       def kill(pid)
@@ -137,7 +152,7 @@ module Ferrum
         output = ""
         start = Utils::ElapsedTime.monotonic_time
         max_time = start + timeout
-        regexp = %r{DevTools listening on (ws://.*)}
+        regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
         while (now = Utils::ElapsedTime.monotonic_time) < max_time
           begin
             output += read_io.read_nonblock(512)
@@ -163,25 +178,37 @@ module Ferrum
         @port = @ws_url.port
       end
 
-      def parse_browser_versions
-        return unless ws_url.is_a?(Addressable::URI)
+      def close_io(*ios)
+        ios.each do |io|
+          io.close unless io.closed?
+        rescue IOError
+          raise unless RUBY_ENGINE == "jruby"
+        end
+      end
+
+      def parse_json_version(url)
+        url = URI.join(url, "/json/version")
+
+        if %w[wss ws].include?(url.scheme)
+          url.scheme = case url.scheme
+                       when "ws"
+                         "http"
+                       when "wss"
+                         "https"
+                       end
+        end
 
-        version_url = URI.parse(ws_url.merge(scheme: "http", path: "/json/version"))
-        response = JSON.parse(::Net::HTTP.get(version_url))
+        response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
 
         @v8_version = response["V8-Version"]
         @browser_version = response["Browser"]
         @webkit_version = response["WebKit-Version"]
         @default_user_agent = response["User-Agent"]
         @protocol_version = response["Protocol-Version"]
-      end
 
-      def close_io(*ios)
-        ios.each do |io|
-          io.close unless io.closed?
-        rescue IOError
-          raise unless RUBY_ENGINE == "jruby"
-        end
+        response
+      rescue StandardError
+        # nop
       end
     end
   end

data/lib/ferrum/browser.rb

@@ -4,11 +4,11 @@ require "base64"
 require "forwardable"
 require "ferrum/page"
 require "ferrum/proxy"
+require "ferrum/client"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
 require "ferrum/browser/options"
 require "ferrum/browser/process"
-require "ferrum/browser/client"
 require "ferrum/browser/binary"
 require "ferrum/browser/version_info"
 
@@ -20,18 +20,20 @@ module Ferrum
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
                 body doctype content=
-                headers cookies network
+                headers cookies network downloads
                 mouse keyboard
-                screenshot pdf mhtml viewport_size
+                screenshot pdf mhtml viewport_size device_pixel_ratio
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
                 on position position=
-                playback_rate playback_rate=] => :page
-    delegate %i[default_user_agent] => :process
+                playback_rate playback_rate=
+                disable_javascript set_viewport resize] => :page
 
-    attr_reader :client, :process, :contexts, :options, :window_size, :base_url
-    attr_accessor :timeout
+    attr_reader :client, :process, :contexts, :options
+
+    delegate %i[timeout timeout= base_url base_url= default_user_agent default_user_agent= extensions] => :options
+    delegate %i[command] => :client
 
     #
     # Initializes the browser.
@@ -45,6 +47,9 @@ module Ferrum
     # @option options [Boolean] :xvfb (false)
     #   Run browser in a virtual framebuffer.
     #
+    # @option options [Boolean] :flatten (true)
+    #   Use one websocket connection to the browser and all the pages in flatten mode.
+    #
     # @option options [(Integer, Integer)] :window_size ([1024, 768])
     #   The dimensions of the browser window in which to test, expressed as a
     #   2-element array, e.g. `[1024, 768]`.
@@ -124,28 +129,9 @@ module Ferrum
       @options = Options.new(options)
       @client = @process = @contexts = nil
 
-      @timeout = @options.timeout
-      @window_size = @options.window_size
-      @base_url = @options.base_url if @options.base_url
-
       start
     end
 
-    #
-    # Sets the base URL.
-    #
-    # @param [String] value
-    #   The new base URL value.
-    #
-    # @raise [ArgumentError] when path is not absolute or doesn't include schema
-    #
-    # @return [Addressable::URI]
-    #   The parsed base URI value.
-    #
-    def base_url=(value)
-      @base_url = options.parse_base_url(value)
-    end
-
     #
     # Creates a new page.
     #
@@ -163,7 +149,7 @@ module Ferrum
                params = {}
 
                if proxy
-                 options.parse_proxy(proxy)
+                 options.validate_proxy(proxy)
                  params.merge!(proxyServer: "#{proxy[:host]}:#{proxy[:port]}")
                  params.merge!(proxyBypassList: proxy[:bypass]) if proxy[:bypass]
                end
@@ -177,17 +163,11 @@ module Ferrum
       block_given? ? yield(page) : page
     ensure
       if block_given?
-        page.close
+        page&.close
         context.dispose if new_context
       end
     end
 
-    def extensions
-      @extensions ||= Array(options.extensions).map do |ext|
-        (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
-      end
-    end
-
     #
     # Evaluate JavaScript to modify things before a page load.
     #
@@ -205,13 +185,6 @@ module Ferrum
       extensions << expression
     end
 
-    def command(*args)
-      @client.command(*args)
-    rescue DeadBrowserError
-      restart
-      raise
-    end
-
     #
     # Closes browser tabs opened by the `Browser` instance.
     #
@@ -227,7 +200,6 @@ module Ferrum
     #   browser.quit
     #
     def reset
-      @window_size = options.window_size
       contexts.reset
     end
 
@@ -237,16 +209,15 @@ module Ferrum
     end
 
     def quit
+      return unless @client
+
+      contexts.close_connections
+
       @client.close
       @process.stop
       @client = @process = @contexts = nil
     end
 
-    def resize(**options)
-      @window_size = [options[:width], options[:height]]
-      page.resize(**options)
-    end
-
     def crash
       command("Browser.crash")
     end
@@ -262,15 +233,26 @@ module Ferrum
       VersionInfo.new(command("Browser.getVersion"))
     end
 
+    def headless_new?
+      process&.command&.headless_new?
+    end
+
     private
 
     def start
       Utils::ElapsedTime.start
-      @process = Process.start(options)
-      @client = Client.new(@process.ws_url, self,
-                           logger: options.logger,
-                           ws_max_receive_size: options.ws_max_receive_size)
-      @contexts = Contexts.new(self)
+      @process = Process.new(options)
+
+      begin
+        @process.start
+        @options.default_user_agent = @process.default_user_agent
+
+        @client = Client.new(@process.ws_url, options)
+        @contexts = Contexts.new(@client)
+      rescue StandardError
+        @process.stop
+        raise
+      end
     end
   end
 end

data/lib/ferrum/client/subscriber.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Client
+    class Subscriber
+      INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
+
+      def initialize
+        @regular = Queue.new
+        @priority = Queue.new
+        @on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
+
+        start
+      end
+
+      def <<(message)
+        if INTERRUPTIONS.include?(message["method"])
+          @priority.push(message)
+        else
+          @regular.push(message)
+        end
+      end
+
+      def on(event, &block)
+        @on[event] << block
+        true
+      end
+
+      def subscribed?(event)
+        @on.key?(event)
+      end
+
+      def close
+        @regular_thread&.kill
+        @priority_thread&.kill
+      end
+
+      def clear(session_id:)
+        @on.delete_if { |k, _| k.match?(session_id) }
+      end
+
+      private
+
+      def start
+        @regular_thread = Utils::Thread.spawn(abort_on_exception: false) do
+          loop do
+            message = @regular.pop
+            break unless message
+
+            call(message)
+          end
+        end
+
+        @priority_thread = Utils::Thread.spawn(abort_on_exception: false) do
+          loop do
+            message = @priority.pop
+            break unless message
+
+            call(message)
+          end
+        end
+      end
+
+      def call(message)
+        method, session_id, params = message.values_at("method", "sessionId", "params")
+        event = SessionClient.event_name(method, session_id)
+
+        total = @on[event].size
+        @on[event].each_with_index do |block, index|
+          # In case of multiple callbacks we provide current index and total
+          block.call(params, index, total)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/{browser → client}/web_socket.rb

@@ -5,18 +5,30 @@ require "socket"
 require "websocket/driver"
 
 module Ferrum
-  class Browser
+  class Client
     class WebSocket
-      WEBSOCKET_BUG_SLEEP = 0.01
+      WEBSOCKET_BUG_SLEEP = 0.05
+      DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
       SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
 
       attr_reader :url, :messages
 
       def initialize(url, max_receive_size, logger)
-        @url      = url
-        @logger   = logger
-        uri       = URI.parse(@url)
-        @sock     = TCPSocket.new(uri.host, uri.port)
+        @url    = url
+        @logger = logger
+        uri     = URI.parse(@url)
+        port    = uri.port || DEFAULT_PORTS[uri.scheme]
+
+        if port == 443
+          tcp = TCPSocket.new(uri.host, port)
+          ssl_context = OpenSSL::SSL::SSLContext.new
+          @sock = OpenSSL::SSL::SSLSocket.new(tcp, ssl_context)
+          @sock.sync_close = true
+          @sock.connect
+        else
+          @sock = TCPSocket.new(uri.host, port)
+        end
+
         max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
         @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
         @messages = Queue.new
@@ -27,21 +39,7 @@ module Ferrum
         @driver.on(:message, &method(:on_message))
         @driver.on(:close,   &method(:on_close))
 
-        @thread = Thread.new do
-          Thread.current.abort_on_exception = true
-          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
-
-          begin
-            loop do
-              data = @sock.readpartial(512)
-              break unless data
-
-              @driver.parse(data)
-            end
-          rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
-            @messages.close
-          end
-        end
+        start
 
         @driver.start
       end
@@ -66,6 +64,7 @@ module Ferrum
 
       def on_close(_event)
         @messages.close
+        @sock.close
         @thread.kill
       end
 
@@ -79,13 +78,28 @@ module Ferrum
 
       def write(data)
         @sock.write(data)
-      rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
+      rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
         @messages.close
       end
 
       def close
         @driver.close
       end
+
+      private
+
+      def start
+        @thread = Utils::Thread.spawn do
+          loop do
+            data = @sock.readpartial(512)
+            break unless data
+
+            @driver.parse(data)
+          end
+        rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
+          @messages.close
+        end
+      end
     end
   end
 end