ferrum 0.7 → 0.10.2

data/lib/ferrum.rb

@@ -10,14 +10,20 @@ module Ferrum
   class NotImplementedError < Error; end
 
   class StatusError < Error
+    def initialize(url, message = nil)
+      super(message || "Request to #{url} failed to reach server, check DNS and server status")
+    end
+  end
+
+  class PendingConnectionsError < StatusError
+    attr_reader :pendings
+
     def initialize(url, pendings = [])
-      message = if pendings.empty?
-                  "Request to #{url} failed to reach server, check DNS and/or server status"
-                else
-                  "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
-                end
+      @pendings = pendings
 
-      super(message)
+      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+
+      super(url, message)
     end
   end
 
@@ -36,12 +42,34 @@ module Ferrum
     end
   end
 
+  class ProcessTimeoutError < Error
+    attr_reader :output
+
+    def initialize(timeout, output)
+      @output = output
+      super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
+    end
+  end
+
   class DeadBrowserError < Error
     def initialize(message = "Browser is dead or given window is closed")
       super
     end
   end
 
+  class NodeIsMovingError < Error
+    def initialize(node, prev, current)
+      @node, @prev, @current = node, prev, current
+      super(message)
+    end
+
+    def message
+      "#{@node.inspect} that you're trying to click is moving, hence " \
+      "we cannot. Previosuly it was at #{@prev.inspect} but now at " \
+      "#{@current.inspect}."
+    end
+  end
+
   class BrowserError < Error
     attr_reader :response
 
@@ -72,8 +100,8 @@ module Ferrum
     attr_reader :class_name, :message
 
     def initialize(response)
-      super
       @class_name, @message = response.values_at("className", "description")
+      super(response.merge("message" => @message))
     end
   end
 

data/lib/ferrum/browser.rb

@@ -4,6 +4,7 @@ require "base64"
 require "forwardable"
 require "ferrum/page"
 require "ferrum/contexts"
+require "ferrum/browser/xvfb"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
 
@@ -16,19 +17,20 @@ module Ferrum
     extend Forwardable
     delegate %i[default_context] => :contexts
     delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[goto back forward refresh reload
-                at_css at_xpath css xpath current_url title body doctype
+    delegate %i[go_to back forward refresh reload stop wait_for_reload
+                at_css at_xpath css xpath current_url current_title url title
+                body doctype set_content
                 headers cookies network
                 mouse keyboard
-                screenshot pdf viewport_size
+                screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
-                evaluate evaluate_on evaluate_async execute
+                evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
-                on] => :page
+                on goto] => :page
     delegate %i[default_user_agent] => :process
 
-    attr_reader :client, :process, :contexts, :logger, :js_errors,
-                :slowmo, :base_url, :options, :window_size
+    attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
+                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
     attr_writer :timeout
 
     def initialize(options = nil)
@@ -39,9 +41,11 @@ module Ferrum
       @original_window_size = @window_size
 
       @options = Hash(options.merge(window_size: @window_size))
-      @logger, @timeout = @options.values_at(:logger, :timeout)
+      @logger, @timeout, @ws_max_receive_size =
+        @options.values_at(:logger, :timeout, :ws_max_receive_size)
       @js_errors = @options.fetch(:js_errors, false)
-      @slowmo = @options[:slowmo].to_i
+      @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+      @slowmo = @options[:slowmo].to_f
 
       if @options.key?(:base_url)
         self.base_url = @options[:base_url]
@@ -114,7 +118,7 @@ module Ferrum
     def start
       Ferrum.started
       @process = Process.start(@options)
-      @client = Client.new(self, @process.ws_url, 0, false)
+      @client = Client.new(self, @process.ws_url)
       @contexts = Contexts.new(self)
     end
   end

data/lib/ferrum/browser/client.rb

@@ -9,12 +9,11 @@ module Ferrum
     class Client
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
-      def initialize(browser, ws_url, start_id = 0, allow_slowmo = true)
-        @command_id = start_id
-        @pendings = Concurrent::Hash.new
+      def initialize(browser, ws_url, id_starts_with: 0)
         @browser = browser
-        @slowmo = @browser.slowmo if allow_slowmo && @browser.slowmo > 0
-        @ws = WebSocket.new(ws_url, @browser.logger)
+        @command_id = id_starts_with
+        @pendings = Concurrent::Hash.new
+        @ws = WebSocket.new(ws_url, @browser.ws_max_receive_size, @browser.logger)
         @subscriber, @interruptor = Subscriber.build(2)
 
         @thread = Thread.new do
@@ -39,7 +38,6 @@ module Ferrum
         pending = Concurrent::IVar.new
         message = build_message(method, params)
         @pendings[message[:id]] = pending
-        sleep(@slowmo) if @slowmo
         @ws.send_message(message)
         data = pending.value!(@browser.timeout)
         @pendings.delete(message[:id])
@@ -60,6 +58,10 @@ module Ferrum
         end
       end
 
+      def subscribed?(event)
+        [@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
+      end
+
       def close
         @ws.close
         # Give a thread some time to handle a tail of messages

data/lib/ferrum/browser/command.rb

@@ -3,53 +3,54 @@
 module Ferrum
   class Browser
     class Command
-      BROWSER_HOST = "127.0.0.1"
-      BROWSER_PORT = "0"
       NOT_FOUND = "Could not find an executable for the browser. Try to make " \
-                  "it available on the PATH or set environment varible for " \
+                  "it available on the PATH or set environment variable for " \
                   "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
 
       # Currently only these browsers support CDP:
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
       def self.build(options, user_data_dir)
-        case options[:browser_name]
-        when :firefox
-          Firefox
-        when :chrome, :opera, :edge, nil
-          Chrome
-        else
-          raise NotImplementedError, "not supported browser"
-        end.new(options, user_data_dir)
+        defaults = case options[:browser_name]
+                   when :firefox
+                     Options::Firefox.options
+                   when :chrome, :opera, :edge, nil
+                     Options::Chrome.options
+                   else
+                     raise NotImplementedError, "not supported browser"
+                   end
+
+        new(defaults, options, user_data_dir)
       end
 
-      attr_reader :path, :flags, :options
+      attr_reader :defaults, :path, :options
 
-      def initialize(options, user_data_dir)
+      def initialize(defaults, options, user_data_dir)
         @flags = {}
+        @defaults = defaults
         @options, @user_data_dir = options, user_data_dir
-        @path = options[:browser_path] || ENV["BROWSER_PATH"] || detect_path
+        @path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
         raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+        merge_options
+      end
 
-        combine_flags
+      def xvfb?
+        !!options[:xvfb]
       end
 
       def to_a
-        [path] + flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
+        [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
       end
 
       private
 
-      def detect_path
-        if Ferrum.mac?
-          self.class::MAC_BIN_PATH.find { |b| File.exist?(b) }
-        else
-          self.class::LINUX_BIN_PATH
-            .find { |b| p = Cliver.detect(b) and break(p) }
+      def merge_options
+        @flags = defaults.merge_required(@flags, options, @user_data_dir)
+
+        unless options[:ignore_default_browser_options]
+          @flags = defaults.merge_default(@flags, options)
         end
-      end
 
-      def combine_flags
-        raise NotImplementedError
+        @flags.merge!(options.fetch(:browser_options, {}))
       end
     end
   end

data/lib/ferrum/browser/options/base.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Ferrum
+  class Browser
+    module Options
+      class Base
+        BROWSER_HOST = "127.0.0.1"
+        BROWSER_PORT = "0"
+
+        include Singleton
+
+        def self.options
+          instance
+        end
+
+        def to_h
+          self.class::DEFAULT_OPTIONS
+        end
+
+        def except(*keys)
+          to_h.reject { |n, _| keys.include?(n) }
+        end
+
+        def detect_path
+          if Ferrum.mac?
+            self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
+          else
+            self.class::LINUX_BIN_PATH.find do |name|
+              path = Cliver.detect(name) and break(path)
+            end
+          end
+        end
+
+        def merge_required(flags, options, user_data_dir)
+          raise NotImplementedError
+        end
+
+        def merge_default(flags, options)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/options/chrome.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module Options
+      class Chrome < Base
+        DEFAULT_OPTIONS = {
+          "headless" => nil,
+          "disable-gpu" => nil,
+          "hide-scrollbars" => nil,
+          "mute-audio" => nil,
+          "enable-automation" => nil,
+          "disable-web-security" => nil,
+          "disable-session-crashed-bubble" => nil,
+          "disable-breakpad" => nil,
+          "disable-sync" => nil,
+          "no-first-run" => nil,
+          "use-mock-keychain" => nil,
+          "keep-alive-for-test" => nil,
+          "disable-popup-blocking" => nil,
+          "disable-extensions" => nil,
+          "disable-hang-monitor" => nil,
+          "disable-features" => "site-per-process,TranslateUI",
+          "disable-translate" => nil,
+          "disable-background-networking" => nil,
+          "enable-features" => "NetworkService,NetworkServiceInProcess",
+          "disable-background-timer-throttling" => nil,
+          "disable-backgrounding-occluded-windows" => nil,
+          "disable-client-side-phishing-detection" => nil,
+          "disable-default-apps" => nil,
+          "disable-dev-shm-usage" => nil,
+          "disable-ipc-flooding-protection" => nil,
+          "disable-prompt-on-repost" => nil,
+          "disable-renderer-backgrounding" => nil,
+          "force-color-profile" => "srgb",
+          "metrics-recording-only" => nil,
+          "safebrowsing-disable-auto-update" => nil,
+          "password-store" => "basic",
+          # Note: --no-sandbox is not needed if you properly setup a user in the container.
+          # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
+          # "no-sandbox" => nil,
+        }.freeze
+
+        MAC_BIN_PATH = [
+          "/Applications/Chromium.app/Contents/MacOS/Chromium",
+          "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+        ].freeze
+        LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
+                            google-chrome chrome chromium-browser
+                            google-chrome-stable].freeze
+
+        def merge_required(flags, options, user_data_dir)
+          port = options.fetch(:port, BROWSER_PORT)
+          host = options.fetch(:host, BROWSER_HOST)
+          flags.merge("remote-debugging-port" => port,
+                      "remote-debugging-address" => host,
+                      # Doesn't work on MacOS, so we need to set it by CDP
+                      "window-size" => options[:window_size].join(","),
+                      "user-data-dir" => user_data_dir)
+        end
+
+        def merge_default(flags, options)
+          unless options.fetch(:headless, true)
+            defaults = except("headless", "disable-gpu")
+          end
+
+          defaults ||= DEFAULT_OPTIONS
+          defaults.merge(flags)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/options/firefox.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module Options
+      class Firefox < Base
+        DEFAULT_OPTIONS = {
+          "headless" => nil,
+        }.freeze
+
+        MAC_BIN_PATH = [
+          "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
+        ].freeze
+        LINUX_BIN_PATH = %w[firefox].freeze
+
+        def merge_required(flags, options, user_data_dir)
+          port = options.fetch(:port, BROWSER_PORT)
+          host = options.fetch(:host, BROWSER_HOST)
+          flags.merge("remote-debugger" => "#{host}:#{port}",
+                      "profile" => user_data_dir)
+        end
+
+        def merge_default(flags, options)
+          unless options.fetch(:headless, true)
+            defaults = except("headless")
+          end
+
+          defaults ||= DEFAULT_OPTIONS
+          defaults.merge(flags)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -6,20 +6,21 @@ require "json"
 require "addressable"
 require "tmpdir"
 require "forwardable"
+require "ferrum/browser/options/base"
+require "ferrum/browser/options/chrome"
+require "ferrum/browser/options/firefox"
 require "ferrum/browser/command"
-require "ferrum/browser/chrome"
-require "ferrum/browser/firefox"
 
 module Ferrum
   class Browser
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
-      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 2).to_i
+      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
 
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
-                  :v8_version, :webkit_version
+                  :v8_version, :webkit_version, :xvfb
 
 
       extend Forwardable
@@ -63,11 +64,13 @@ module Ferrum
           return
         end
 
+        @pid = @xvfb = @user_data_dir = nil
         @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
 
-        tmpdir = Dir.mktmpdir
+        tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
+        @user_data_dir = tmpdir
         @command = Command.build(options, tmpdir)
       end
 
@@ -81,7 +84,12 @@ module Ferrum
           process_options[:pgroup] = true unless Ferrum.windows?
           process_options[:out] = process_options[:err] = write_io
 
-          @pid = ::Process.spawn(*@command.to_a, process_options)
+          if @command.xvfb?
+            @xvfb = Xvfb.start(@command.options)
+            ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
+          end
+
+          @pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
@@ -92,7 +100,12 @@ module Ferrum
       end
 
       def stop
-        kill if @pid
+        if @pid
+          kill(@pid)
+          kill(@xvfb.pid) if @xvfb&.pid
+          @pid = nil
+        end
+
         remove_user_data_dir if @user_data_dir
         ObjectSpace.undefine_finalizer(self)
       end
@@ -104,9 +117,8 @@ module Ferrum
 
       private
 
-      def kill
-        self.class.process_killer(@pid).call
-        @pid = nil
+      def kill(pid)
+        self.class.process_killer(pid).call
       end
 
       def remove_user_data_dir
@@ -133,8 +145,8 @@ module Ferrum
         end
 
         unless ws_url
-          @logger.puts output if @logger
-          raise "Browser process did not produce websocket url within #{timeout} seconds"
+          @logger.puts(output) if @logger
+          raise ProcessTimeoutError.new(timeout, output)
         end
       end