ferrum 0.14 → 0.16

data/lib/ferrum/browser/command.rb

@@ -47,6 +47,10 @@ module Ferrum
         [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
       end
 
+      def to_s
+        to_a.join(" \\ \n  ")
+      end
+
       private
 
       def merge_options

data/lib/ferrum/browser/options/chrome.rb

@@ -6,7 +6,6 @@ module Ferrum
       class Chrome < Base
         DEFAULT_OPTIONS = {
           "headless" => nil,
-          "disable-gpu" => nil,
           "hide-scrollbars" => nil,
           "mute-audio" => nil,
           "enable-automation" => nil,
@@ -38,7 +37,8 @@ module Ferrum
           "metrics-recording-only" => nil,
           "safebrowsing-disable-auto-update" => nil,
           "password-store" => "basic",
-          "no-startup-window" => nil
+          "no-startup-window" => nil,
+          "remote-allow-origins" => "*"
           # NOTE: --no-sandbox is not needed if you properly setup a user in the container.
           # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
           # "no-sandbox" => nil,
@@ -63,7 +63,6 @@ module Ferrum
         def merge_required(flags, options, user_data_dir)
           flags = flags.merge("remote-debugging-port" => options.port,
                               "remote-debugging-address" => options.host,
-                              # Doesn't work on MacOS, so we need to set it by CDP
                               "window-size" => options.window_size&.join(","),
                               "user-data-dir" => user_data_dir)
 
@@ -84,6 +83,12 @@ module Ferrum
                      end
 
           defaults ||= DEFAULT_OPTIONS
+          # On Windows, the --disable-gpu flag is a temporary work around for a few bugs.
+          # See https://bugs.chromium.org/p/chromium/issues/detail?id=737678 for more information.
+          defaults = defaults.merge("disable-gpu" => nil) if Utils::Platform.windows?
+          # Use Metal on Apple Silicon
+          # https://github.com/google/angle#platform-support-via-backing-renderers
+          defaults = defaults.merge("use-angle" => "metal") if Utils::Platform.mac_arm?
           defaults.merge(flags)
         end
       end

data/lib/ferrum/browser/options.rb

@@ -3,7 +3,6 @@
 module Ferrum
   class Browser
     class Options
-      HEADLESS = true
       BROWSER_PORT = "0"
       BROWSER_HOST = "127.0.0.1"
       WINDOW_SIZE = [1024, 768].freeze
@@ -12,55 +11,56 @@ module Ferrum
       PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
       DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
 
-      attr_reader :window_size, :timeout, :logger, :ws_max_receive_size,
+      attr_reader :window_size, :logger, :ws_max_receive_size,
                   :js_errors, :base_url, :slowmo, :pending_connection_errors,
-                  :url, :env, :process_timeout, :browser_name, :browser_path,
-                  :save_path, :extensions, :proxy, :port, :host, :headless,
-                  :ignore_default_browser_options, :browser_options, :xvfb
+                  :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
+                  :save_path, :proxy, :port, :host, :headless, :browser_options,
+                  :ignore_default_browser_options, :xvfb, :flatten
+      attr_accessor :timeout, :default_user_agent
 
       def initialize(options = nil)
         @options = Hash(options&.dup)
+
         @port = @options.fetch(:port, BROWSER_PORT)
         @host = @options.fetch(:host, BROWSER_HOST)
         @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
         @window_size = @options.fetch(:window_size, WINDOW_SIZE)
         @js_errors = @options.fetch(:js_errors, false)
-        @headless = @options.fetch(:headless, HEADLESS)
+        @headless = @options.fetch(:headless, true)
+        @flatten = @options.fetch(:flatten, true)
         @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
-        @browser_options = @options.fetch(:browser_options, {})
         @slowmo = @options[:slowmo].to_f
 
-        @ws_max_receive_size, @env, @browser_name, @browser_path,
-          @save_path, @extensions, @ignore_default_browser_options, @xvfb = @options.values_at(
-            :ws_max_receive_size, :env, :browser_name, :browser_path, :save_path, :extensions,
-            :ignore_default_browser_options, :xvfb
-          )
+        @env = @options[:env]
+        @xvfb = @options[:xvfb]
+        @save_path = @options[:save_path]
+        @browser_name = @options[:browser_name]
+        @browser_path = @options[:browser_path]
+        @ws_max_receive_size = @options[:ws_max_receive_size]
+        @ignore_default_browser_options = @options[:ignore_default_browser_options]
 
-        @options[:window_size] = @window_size
-        @proxy = parse_proxy(@options[:proxy])
+        @proxy = validate_proxy(@options[:proxy])
         @logger = parse_logger(@options[:logger])
         @base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
         @url = @options[:url].to_s if @options[:url]
+        @ws_url = @options[:ws_url].to_s if @options[:ws_url]
 
-        @options.freeze
-        @browser_options.freeze
+        @options = @options.merge(window_size: @window_size).freeze
+        @browser_options = @options.fetch(:browser_options, {}).freeze
       end
 
-      def to_h
-        @options
+      def base_url=(value)
+        @base_url = parse_base_url(value)
       end
 
-      def parse_base_url(value)
-        parsed = Addressable::URI.parse(value)
-        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
-          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+      def extensions
+        @extensions ||= Array(@options[:extensions]).map do |extension|
+          (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
         end
-
-        parsed
       end
 
-      def parse_proxy(options)
+      def validate_proxy(options)
         return unless options
 
         raise ArgumentError, "proxy options must be a Hash" unless options.is_a?(Hash)
@@ -72,6 +72,10 @@ module Ferrum
         options
       end
 
+      def to_h
+        @options
+      end
+
       private
 
       def parse_logger(logger)
@@ -79,6 +83,15 @@ module Ferrum
 
         !logger && DEBUG_MODE ? $stdout.tap { |s| s.sync = true } : logger
       end
+
+      def parse_base_url(value)
+        parsed = Addressable::URI.parse(value)
+        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
+          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+        end
+
+        parsed
+      end
     end
   end
 end

data/lib/ferrum/browser/process.rb

@@ -9,6 +9,8 @@ require "ferrum/browser/options/base"
 require "ferrum/browser/options/chrome"
 require "ferrum/browser/options/firefox"
 require "ferrum/browser/command"
+require "ferrum/utils/elapsed_time"
+require "ferrum/utils/platform"
 
 module Ferrum
   class Browser
@@ -62,11 +64,11 @@ module Ferrum
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
 
-        if options.url
-          url = URI.join(options.url, "/json/version")
-          response = JSON.parse(::Net::HTTP.get(url))
-          self.ws_url = response["webSocketDebuggerUrl"]
-          parse_browser_versions
+        if options.ws_url || options.url
+          # `:ws_url` option is higher priority than `:url`, parse versions
+          # and use it as a ws_url, otherwise use what has been parsed.
+          response = parse_json_version(options.ws_url || options.url)
+          self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
           return
         end
 
@@ -100,7 +102,7 @@ module Ferrum
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
-          parse_browser_versions
+          parse_json_version(ws_url)
         ensure
           close_io(read_io, write_io)
         end
@@ -122,6 +124,17 @@ module Ferrum
         start
       end
 
+      def inspect
+        "#<#{self.class} " \
+          "@user_data_dir=#{@user_data_dir.inspect} " \
+          "@command=#<#{@command.class}:#{@command.object_id}> " \
+          "@default_user_agent=#{@default_user_agent.inspect} " \
+          "@ws_url=#{@ws_url.inspect} " \
+          "@v8_version=#{@v8_version.inspect} " \
+          "@browser_version=#{@browser_version.inspect} " \
+          "@webkit_version=#{@webkit_version.inspect}>"
+      end
+
       private
 
       def kill(pid)
@@ -163,25 +176,37 @@ module Ferrum
         @port = @ws_url.port
       end
 
-      def parse_browser_versions
-        return unless ws_url.is_a?(Addressable::URI)
+      def close_io(*ios)
+        ios.each do |io|
+          io.close unless io.closed?
+        rescue IOError
+          raise unless RUBY_ENGINE == "jruby"
+        end
+      end
 
-        version_url = URI.parse(ws_url.merge(scheme: "http", path: "/json/version"))
-        response = JSON.parse(::Net::HTTP.get(version_url))
+      def parse_json_version(url)
+        url = URI.join(url, "/json/version")
+
+        if %w[wss ws].include?(url.scheme)
+          url.scheme = case url.scheme
+                       when "ws"
+                         "http"
+                       when "wss"
+                         "https"
+                       end
+        end
+
+        response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
 
         @v8_version = response["V8-Version"]
         @browser_version = response["Browser"]
         @webkit_version = response["WebKit-Version"]
         @default_user_agent = response["User-Agent"]
         @protocol_version = response["Protocol-Version"]
-      end
 
-      def close_io(*ios)
-        ios.each do |io|
-          io.close unless io.closed?
-        rescue IOError
-          raise unless RUBY_ENGINE == "jruby"
-        end
+        response
+      rescue JSON::ParserError
+        # nop
       end
     end
   end

data/lib/ferrum/browser.rb

@@ -4,11 +4,11 @@ require "base64"
 require "forwardable"
 require "ferrum/page"
 require "ferrum/proxy"
+require "ferrum/client"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
 require "ferrum/browser/options"
 require "ferrum/browser/process"
-require "ferrum/browser/client"
 require "ferrum/browser/binary"
 require "ferrum/browser/version_info"
 
@@ -20,18 +20,20 @@ module Ferrum
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
                 body doctype content=
-                headers cookies network
+                headers cookies network downloads
                 mouse keyboard
                 screenshot pdf mhtml viewport_size device_pixel_ratio
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
                 on position position=
-                playback_rate playback_rate=] => :page
-    delegate %i[default_user_agent] => :process
+                playback_rate playback_rate=
+                disable_javascript set_viewport resize] => :page
 
-    attr_reader :client, :process, :contexts, :options, :window_size, :base_url
-    attr_accessor :timeout
+    attr_reader :client, :process, :contexts, :options
+
+    delegate %i[timeout timeout= base_url base_url= default_user_agent default_user_agent= extensions] => :options
+    delegate %i[command] => :client
 
     #
     # Initializes the browser.
@@ -45,6 +47,9 @@ module Ferrum
     # @option options [Boolean] :xvfb (false)
     #   Run browser in a virtual framebuffer.
     #
+    # @option options [Boolean] :flatten (true)
+    #   Use one websocket connection to the browser and all the pages in flatten mode.
+    #
     # @option options [(Integer, Integer)] :window_size ([1024, 768])
     #   The dimensions of the browser window in which to test, expressed as a
     #   2-element array, e.g. `[1024, 768]`.
@@ -124,28 +129,9 @@ module Ferrum
       @options = Options.new(options)
       @client = @process = @contexts = nil
 
-      @timeout = @options.timeout
-      @window_size = @options.window_size
-      @base_url = @options.base_url if @options.base_url
-
       start
     end
 
-    #
-    # Sets the base URL.
-    #
-    # @param [String] value
-    #   The new base URL value.
-    #
-    # @raise [ArgumentError] when path is not absolute or doesn't include schema
-    #
-    # @return [Addressable::URI]
-    #   The parsed base URI value.
-    #
-    def base_url=(value)
-      @base_url = options.parse_base_url(value)
-    end
-
     #
     # Creates a new page.
     #
@@ -163,7 +149,7 @@ module Ferrum
                params = {}
 
                if proxy
-                 options.parse_proxy(proxy)
+                 options.validate_proxy(proxy)
                  params.merge!(proxyServer: "#{proxy[:host]}:#{proxy[:port]}")
                  params.merge!(proxyBypassList: proxy[:bypass]) if proxy[:bypass]
                end
@@ -182,12 +168,6 @@ module Ferrum
       end
     end
 
-    def extensions
-      @extensions ||= Array(options.extensions).map do |ext|
-        (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
-      end
-    end
-
     #
     # Evaluate JavaScript to modify things before a page load.
     #
@@ -205,13 +185,6 @@ module Ferrum
       extensions << expression
     end
 
-    def command(*args)
-      @client.command(*args)
-    rescue DeadBrowserError
-      restart
-      raise
-    end
-
     #
     # Closes browser tabs opened by the `Browser` instance.
     #
@@ -227,7 +200,6 @@ module Ferrum
     #   browser.quit
     #
     def reset
-      @window_size = options.window_size
       contexts.reset
     end
 
@@ -239,20 +211,29 @@ module Ferrum
     def quit
       return unless @client
 
+      contexts.close_connections
+
       @client.close
       @process.stop
       @client = @process = @contexts = nil
     end
 
-    def resize(**options)
-      @window_size = [options[:width], options[:height]]
-      page.resize(**options)
-    end
-
+    #
+    # Crashes browser.
+    #
     def crash
       command("Browser.crash")
     end
 
+    #
+    # Close browser gracefully.
+    #
+    # You should clean up resources/connections in ruby world manually, it's only a CDP command.
+    #
+    def close
+      command("Browser.close")
+    end
+
     #
     # Gets the version information from the browser.
     #
@@ -272,11 +253,18 @@ module Ferrum
 
     def start
       Utils::ElapsedTime.start
-      @process = Process.start(options)
-      @client = Client.new(@process.ws_url, self,
-                           logger: options.logger,
-                           ws_max_receive_size: options.ws_max_receive_size)
-      @contexts = Contexts.new(self)
+      @process = Process.new(options)
+
+      begin
+        @process.start
+        @options.default_user_agent = @process.default_user_agent
+
+        @client = Client.new(@process.ws_url, options)
+        @contexts = Contexts.new(@client)
+      rescue StandardError
+        @process.stop
+        raise
+      end
     end
   end
 end

data/lib/ferrum/client/subscriber.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Client
+    class Subscriber
+      INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
+
+      def initialize
+        @regular = Queue.new
+        @priority = Queue.new
+        @on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
+
+        start
+      end
+
+      def <<(message)
+        if INTERRUPTIONS.include?(message["method"])
+          @priority.push(message)
+        else
+          @regular.push(message)
+        end
+      end
+
+      def on(event, &block)
+        @on[event] << block
+        true
+      end
+
+      def subscribed?(event)
+        @on.key?(event)
+      end
+
+      def close
+        @regular_thread&.kill
+        @priority_thread&.kill
+      end
+
+      def clear(session_id:)
+        @on.delete_if { |k, _| k.match?(session_id) }
+      end
+
+      private
+
+      def start
+        @regular_thread = Utils::Thread.spawn(abort_on_exception: false) do
+          loop do
+            message = @regular.pop
+            break unless message
+
+            call(message)
+          end
+        end
+
+        @priority_thread = Utils::Thread.spawn(abort_on_exception: false) do
+          loop do
+            message = @priority.pop
+            break unless message
+
+            call(message)
+          end
+        end
+      end
+
+      def call(message)
+        method, session_id, params = message.values_at("method", "sessionId", "params")
+        event = SessionClient.event_name(method, session_id)
+
+        total = @on[event].size
+        @on[event].each_with_index do |block, index|
+          # In case of multiple callbacks we provide current index and total
+          block.call(params, index, total)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/client/web_socket.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+require "websocket/driver"
+
+module Ferrum
+  class Client
+    class WebSocket
+      WEBSOCKET_BUG_SLEEP = 0.05
+      DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
+      SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
+
+      attr_reader :url, :messages
+
+      def initialize(url, max_receive_size, logger)
+        @url    = url
+        @logger = logger
+        uri     = URI.parse(@url)
+        port    = uri.port || DEFAULT_PORTS[uri.scheme]
+
+        if port == 443
+          tcp = TCPSocket.new(uri.host, port)
+          ssl_context = OpenSSL::SSL::SSLContext.new
+          @sock = OpenSSL::SSL::SSLSocket.new(tcp, ssl_context)
+          @sock.sync_close = true
+          @sock.connect
+        else
+          @sock = TCPSocket.new(uri.host, port)
+        end
+
+        max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
+        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        @messages = Queue.new
+
+        @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
+
+        @driver.on(:open,    &method(:on_open))
+        @driver.on(:message, &method(:on_message))
+        @driver.on(:close,   &method(:on_close))
+
+        start
+
+        @driver.start
+      end
+
+      def on_open(_event)
+        # https://github.com/faye/websocket-driver-ruby/issues/46
+        sleep(WEBSOCKET_BUG_SLEEP)
+      end
+
+      def on_message(event)
+        data = safely_parse_json(event.data)
+        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
+        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
+        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
+        # thread and the application.
+        @messages.push(data) if data
+
+        output = event.data
+        if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
+          @screenshot_commands.delete(data&.dig("id"))
+          output.sub!(/{"data":"[^"]*"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
+        end
+
+        @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
+      end
+
+      def on_close(_event)
+        @messages.close
+        @sock.close
+        @thread.kill
+      end
+
+      def send_message(data)
+        @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
+
+        json = data.to_json
+        @driver.text(json)
+        @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
+      end
+
+      def write(data)
+        @sock.write(data)
+      rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
+        @messages.close
+      end
+
+      def close
+        @driver.close
+      end
+
+      private
+
+      def start
+        @thread = Utils::Thread.spawn do
+          loop do
+            data = @sock.readpartial(512)
+            break unless data
+
+            @driver.parse(data)
+          end
+        rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
+          @messages.close
+        end
+      end
+
+      def safely_parse_json(data)
+        JSON.parse(data, max_nesting: false)
+      rescue JSON::NestingError
+        # nop
+      rescue JSON::ParserError
+        safely_parse_escaped_json(data)
+      end
+
+      def safely_parse_escaped_json(data)
+        unescaped_unicode =
+          data.gsub(/\\u([\da-fA-F]{4})/) { |_| [::Regexp.last_match(1)].pack("H*").unpack("n*").pack("U*") }
+        escaped_data = unescaped_unicode.encode("UTF-8", "UTF-8", undef: :replace, invalid: :replace, replace: "?")
+        JSON.parse(escaped_data, max_nesting: false)
+      rescue JSON::ParserError
+        # nop
+      end
+    end
+  end
+end