ferrum 0.6.2 → 0.10.1

data/lib/ferrum.rb

@@ -10,8 +10,20 @@ module Ferrum
   class NotImplementedError < Error; end
 
   class StatusError < Error
-    def initialize(url)
-      super("Request to #{url} failed to reach server, check DNS and/or server status")
+    def initialize(url, message = nil)
+      super(message || "Request to #{url} failed to reach server, check DNS and server status")
+    end
+  end
+
+  class PendingConnectionsError < StatusError
+    attr_reader :pendings
+
+    def initialize(url, pendings = [])
+      @pendings = pendings
+
+      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+
+      super(url, message)
     end
   end
 
@@ -30,12 +42,34 @@ module Ferrum
     end
   end
 
+  class ProcessTimeoutError < Error
+    attr_reader :output
+
+    def initialize(timeout, output)
+      @output = output
+      super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
+    end
+  end
+
   class DeadBrowserError < Error
-    def initialize(message = "Browser is dead")
+    def initialize(message = "Browser is dead or given window is closed")
       super
     end
   end
 
+  class NodeIsMovingError < Error
+    def initialize(node, prev, current)
+      @node, @prev, @current = node, prev, current
+      super(message)
+    end
+
+    def message
+      "#{@node.inspect} that you're trying to click is moving, hence " \
+      "we cannot. Previosuly it was at #{@prev.inspect} but now at " \
+      "#{@current.inspect}."
+    end
+  end
+
   class BrowserError < Error
     attr_reader :response
 
@@ -66,8 +100,8 @@ module Ferrum
     attr_reader :class_name, :message
 
     def initialize(response)
-      super
       @class_name, @message = response.values_at("className", "description")
+      super(response.merge("message" => @message))
     end
   end
 

data/lib/ferrum/browser.rb

@@ -4,6 +4,7 @@ require "base64"
 require "forwardable"
 require "ferrum/page"
 require "ferrum/contexts"
+require "ferrum/browser/xvfb"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
 
@@ -16,18 +17,20 @@ module Ferrum
     extend Forwardable
     delegate %i[default_context] => :contexts
     delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[goto back forward refresh
-                at_css at_xpath css xpath current_url title body doctype
+    delegate %i[go_to back forward refresh reload stop wait_for_reload
+                at_css at_xpath css xpath current_url current_title url title
+                body doctype set_content
                 headers cookies network
                 mouse keyboard
-                screenshot pdf viewport_size
+                screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
-                evaluate evaluate_on evaluate_async execute
-                add_script_tag add_style_tag
-                on] => :page
+                evaluate evaluate_on evaluate_async execute evaluate_func
+                add_script_tag add_style_tag bypass_csp
+                on goto] => :page
+    delegate %i[default_user_agent] => :process
 
-    attr_reader :client, :process, :contexts, :logger, :js_errors,
-                :slowmo, :base_url, :options, :window_size
+    attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
+                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
     attr_writer :timeout
 
     def initialize(options = nil)
@@ -38,9 +41,11 @@ module Ferrum
       @original_window_size = @window_size
 
       @options = Hash(options.merge(window_size: @window_size))
-      @logger, @timeout = @options.values_at(:logger, :timeout)
+      @logger, @timeout, @ws_max_receive_size =
+        @options.values_at(:logger, :timeout, :ws_max_receive_size)
       @js_errors = @options.fetch(:js_errors, false)
-      @slowmo = @options[:slowmo].to_i
+      @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+      @slowmo = @options[:slowmo].to_f
 
       if @options.key?(:base_url)
         self.base_url = @options[:base_url]
@@ -67,7 +72,9 @@ module Ferrum
     end
 
     def extensions
-      @extensions ||= Array(@options[:extensions]).map { |p| File.read(p) }
+      @extensions ||= Array(@options[:extensions]).map do |ext|
+        (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
+      end
     end
 
     def timeout
@@ -111,7 +118,7 @@ module Ferrum
     def start
       Ferrum.started
       @process = Process.start(@options)
-      @client = Client.new(self, @process.ws_url, 0, false)
+      @client = Client.new(self, @process.ws_url)
       @contexts = Contexts.new(self)
     end
   end

data/lib/ferrum/browser/client.rb

@@ -7,20 +7,25 @@ require "ferrum/browser/web_socket"
 module Ferrum
   class Browser
     class Client
-      def initialize(browser, ws_url, start_id = 0, allow_slowmo = true)
-        @command_id = start_id
-        @pendings = Concurrent::Hash.new
+      INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
+
+      def initialize(browser, ws_url, id_starts_with: 0)
         @browser = browser
-        @slowmo = @browser.slowmo if allow_slowmo && @browser.slowmo > 0
-        @ws = WebSocket.new(ws_url, @browser.logger)
-        @subscriber = Subscriber.new
+        @command_id = id_starts_with
+        @pendings = Concurrent::Hash.new
+        @ws = WebSocket.new(ws_url, @browser.ws_max_receive_size, @browser.logger)
+        @subscriber, @interruptor = Subscriber.build(2)
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
+          if Thread.current.respond_to?(:report_on_exception=)
+            Thread.current.report_on_exception = true
+          end
 
           while message = @ws.messages.pop
-            if message.key?("method")
+            if INTERRUPTIONS.include?(message["method"])
+              @interruptor.async.call(message)
+            elsif message.key?("method")
               @subscriber.async.call(message)
             else
               @pendings[message["id"]]&.set(message)
@@ -33,7 +38,6 @@ module Ferrum
         pending = Concurrent::IVar.new
         message = build_message(method, params)
         @pendings[message[:id]] = pending
-        sleep(@slowmo) if @slowmo
         @ws.send_message(message)
         data = pending.value!(@browser.timeout)
         @pendings.delete(message[:id])
@@ -46,7 +50,16 @@ module Ferrum
       end
 
       def on(event, &block)
-        @subscriber.on(event, &block)
+        case event
+        when *INTERRUPTIONS
+          @interruptor.on(event, &block)
+        else
+          @subscriber.on(event, &block)
+        end
+      end
+
+      def subscribed?(event)
+        [@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
       end
 
       def close

data/lib/ferrum/browser/command.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Command
+      NOT_FOUND = "Could not find an executable for the browser. Try to make " \
+                  "it available on the PATH or set environment variable for " \
+                  "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
+
+      # Currently only these browsers support CDP:
+      # https://github.com/cyrus-and/chrome-remote-interface#implementations
+      def self.build(options, user_data_dir)
+        defaults = case options[:browser_name]
+                   when :firefox
+                     Options::Firefox.options
+                   when :chrome, :opera, :edge, nil
+                     Options::Chrome.options
+                   else
+                     raise NotImplementedError, "not supported browser"
+                   end
+
+        new(defaults, options, user_data_dir)
+      end
+
+      attr_reader :defaults, :path, :options
+
+      def initialize(defaults, options, user_data_dir)
+        @flags = {}
+        @defaults = defaults
+        @options, @user_data_dir = options, user_data_dir
+        @path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
+        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+        merge_options
+      end
+
+      def xvfb?
+        !!options[:xvfb]
+      end
+
+      def to_a
+        [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
+      end
+
+      private
+
+      def merge_options
+        @flags = defaults.merge_required(@flags, options, @user_data_dir)
+
+        unless options[:ignore_default_browser_options]
+          @flags = defaults.merge_default(@flags, options)
+        end
+
+        @flags.merge!(options.fetch(:browser_options, {}))
+      end
+    end
+  end
+end

data/lib/ferrum/browser/options/base.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Ferrum
+  class Browser
+    module Options
+      class Base
+        BROWSER_HOST = "127.0.0.1"
+        BROWSER_PORT = "0"
+
+        include Singleton
+
+        def self.options
+          instance
+        end
+
+        def to_h
+          self.class::DEFAULT_OPTIONS
+        end
+
+        def except(*keys)
+          to_h.reject { |n, _| keys.include?(n) }
+        end
+
+        def detect_path
+          if Ferrum.mac?
+            self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
+          else
+            self.class::LINUX_BIN_PATH.find do |name|
+              path = Cliver.detect(name) and break(path)
+            end
+          end
+        end
+
+        def merge_required(flags, options, user_data_dir)
+          raise NotImplementedError
+        end
+
+        def merge_default(flags, options)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/options/chrome.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module Options
+      class Chrome < Base
+        DEFAULT_OPTIONS = {
+          "headless" => nil,
+          "disable-gpu" => nil,
+          "hide-scrollbars" => nil,
+          "mute-audio" => nil,
+          "enable-automation" => nil,
+          "disable-web-security" => nil,
+          "disable-session-crashed-bubble" => nil,
+          "disable-breakpad" => nil,
+          "disable-sync" => nil,
+          "no-first-run" => nil,
+          "use-mock-keychain" => nil,
+          "keep-alive-for-test" => nil,
+          "disable-popup-blocking" => nil,
+          "disable-extensions" => nil,
+          "disable-hang-monitor" => nil,
+          "disable-features" => "site-per-process,TranslateUI",
+          "disable-translate" => nil,
+          "disable-background-networking" => nil,
+          "enable-features" => "NetworkService,NetworkServiceInProcess",
+          "disable-background-timer-throttling" => nil,
+          "disable-backgrounding-occluded-windows" => nil,
+          "disable-client-side-phishing-detection" => nil,
+          "disable-default-apps" => nil,
+          "disable-dev-shm-usage" => nil,
+          "disable-ipc-flooding-protection" => nil,
+          "disable-prompt-on-repost" => nil,
+          "disable-renderer-backgrounding" => nil,
+          "force-color-profile" => "srgb",
+          "metrics-recording-only" => nil,
+          "safebrowsing-disable-auto-update" => nil,
+          "password-store" => "basic",
+          # Note: --no-sandbox is not needed if you properly setup a user in the container.
+          # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
+          # "no-sandbox" => nil,
+        }.freeze
+
+        MAC_BIN_PATH = [
+          "/Applications/Chromium.app/Contents/MacOS/Chromium",
+          "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+        ].freeze
+        LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
+                            google-chrome chrome chromium-browser
+                            google-chrome-stable].freeze
+
+        def merge_required(flags, options, user_data_dir)
+          port = options.fetch(:port, BROWSER_PORT)
+          host = options.fetch(:host, BROWSER_HOST)
+          flags.merge("remote-debugging-port" => port,
+                      "remote-debugging-address" => host,
+                      # Doesn't work on MacOS, so we need to set it by CDP
+                      "window-size" => options[:window_size].join(","),
+                      "user-data-dir" => user_data_dir)
+        end
+
+        def merge_default(flags, options)
+          unless options.fetch(:headless, true)
+            defaults = except("headless", "disable-gpu")
+          end
+
+          defaults ||= DEFAULT_OPTIONS
+          defaults.merge(flags)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/options/firefox.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module Options
+      class Firefox < Base
+        DEFAULT_OPTIONS = {
+          "headless" => nil,
+        }.freeze
+
+        MAC_BIN_PATH = [
+          "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
+        ].freeze
+        LINUX_BIN_PATH = %w[firefox].freeze
+
+        def merge_required(flags, options, user_data_dir)
+          port = options.fetch(:port, BROWSER_PORT)
+          host = options.fetch(:host, BROWSER_HOST)
+          flags.merge("remote-debugger" => "#{host}:#{port}",
+                      "profile" => user_data_dir)
+        end
+
+        def merge_default(flags, options)
+          unless options.fetch(:headless, true)
+            defaults = except("headless")
+          end
+
+          defaults ||= DEFAULT_OPTIONS
+          defaults.merge(flags)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -5,59 +5,26 @@ require "net/http"
 require "json"
 require "addressable"
 require "tmpdir"
+require "forwardable"
+require "ferrum/browser/options/base"
+require "ferrum/browser/options/chrome"
+require "ferrum/browser/options/firefox"
+require "ferrum/browser/command"
 
 module Ferrum
   class Browser
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
-      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 2).to_i
-      BROWSER_PATH = ENV["BROWSER_PATH"]
-      BROWSER_HOST = "127.0.0.1"
-      BROWSER_PORT = "0"
-      DEFAULT_OPTIONS = {
-        "headless" => nil,
-        "disable-gpu" => nil,
-        "hide-scrollbars" => nil,
-        "mute-audio" => nil,
-        "enable-automation" => nil,
-        "disable-web-security" => nil,
-        "disable-session-crashed-bubble" => nil,
-        "disable-breakpad" => nil,
-        "disable-sync" => nil,
-        "no-first-run" => nil,
-        "use-mock-keychain" => nil,
-        "keep-alive-for-test" => nil,
-        "disable-popup-blocking" => nil,
-        "disable-extensions" => nil,
-        "disable-hang-monitor" => nil,
-        "disable-features" => "site-per-process,TranslateUI",
-        "disable-translate" => nil,
-        "disable-background-networking" => nil,
-        "enable-features" => "NetworkService,NetworkServiceInProcess",
-        "disable-background-timer-throttling" => nil,
-        "disable-backgrounding-occluded-windows" => nil,
-        "disable-client-side-phishing-detection" => nil,
-        "disable-default-apps" => nil,
-        "disable-dev-shm-usage" => nil,
-        "disable-ipc-flooding-protection" => nil,
-        "disable-prompt-on-repost" => nil,
-        "disable-renderer-backgrounding" => nil,
-        "force-color-profile" => "srgb",
-        "metrics-recording-only" => nil,
-        "safebrowsing-disable-auto-update" => nil,
-        "password-store" => "basic",
-        # Note: --no-sandbox is not needed if you properly setup a user in the container.
-        # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
-        # "no-sandbox" => nil,
-      }.freeze
-
-      NOT_FOUND = "Could not find an executable for chrome. Try to make it " \
-                  "available on the PATH or set environment varible for " \
-                  "example BROWSER_PATH=\"/Applications/Chromium.app/Contents/MacOS/Chromium\""
-
-
-      attr_reader :host, :port, :ws_url, :pid, :path, :options, :cmd
+      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
+
+      attr_reader :host, :port, :ws_url, :pid, :command,
+                  :default_user_agent, :browser_version, :protocol_version,
+                  :v8_version, :webkit_version, :xvfb
+
+
+      extend Forwardable
+      delegate path: :command
 
       def self.start(*args)
         new(*args).tap(&:start)
@@ -85,65 +52,26 @@ module Ferrum
       end
 
       def self.directory_remover(path)
-        proc do
-          begin
-            FileUtils.remove_entry(path)
-          rescue Errno::ENOENT
-          end
-        end
-      end
-
-      def self.detect_browser_path
-        if RUBY_PLATFORM.include?("darwin")
-          [
-            "/Applications/Chromium.app/Contents/MacOS/Chromium",
-            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
-          ].find { |path| File.exist?(path) }
-        else
-          %w[chromium google-chrome-unstable google-chrome-beta google-chrome chrome chromium-browser google-chrome-stable].reduce(nil) do |path, exe|
-            path = Cliver.detect(exe)
-            break path if path
-          end
-        end
+        proc { FileUtils.remove_entry(path) rescue Errno::ENOENT }
       end
 
       def initialize(options)
-        @options = {}
-
-        @path = options[:browser_path] || BROWSER_PATH || self.class.detect_browser_path
-
         if options[:url]
           url = URI.join(options[:url].to_s, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
           set_ws_url(response["webSocketDebuggerUrl"])
+          parse_browser_versions
           return
         end
 
-        # Doesn't work on MacOS, so we need to set it by CDP as well
-        @options.merge!("window-size" => options[:window_size].join(","))
-
-        port = options.fetch(:port, BROWSER_PORT)
-        @options.merge!("remote-debugging-port" => port)
-
-        host = options.fetch(:host, BROWSER_HOST)
-        @options.merge!("remote-debugging-address" => host)
-
-        @temp_user_data_dir = Dir.mktmpdir
-        ObjectSpace.define_finalizer(self, self.class.directory_remover(@temp_user_data_dir))
-        @options.merge!("user-data-dir" => @temp_user_data_dir)
-
-        @options = DEFAULT_OPTIONS.merge(@options)
-
-        unless options.fetch(:headless, true)
-          @options.delete("headless")
-          @options.delete("disable-gpu")
-        end
-
+        @pid = @xvfb = @user_data_dir = nil
+        @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
 
-        @options.merge!(options.fetch(:browser_options, {}))
-
-        @logger = options[:logger]
+        tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
+        ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
+        @user_data_dir = tmpdir
+        @command = Command.build(options, tmpdir)
       end
 
       def start
@@ -156,21 +84,29 @@ module Ferrum
           process_options[:pgroup] = true unless Ferrum.windows?
           process_options[:out] = process_options[:err] = write_io
 
-          raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+          if @command.xvfb?
+            @xvfb = Xvfb.start(@command.options)
+            ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
+          end
 
-          @cmd = [@path] + @options.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
-          @pid = ::Process.spawn(*@cmd, process_options)
+          @pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
+          parse_browser_versions
         ensure
           close_io(read_io, write_io)
         end
       end
 
       def stop
-        kill if @pid
-        remove_temp_user_data_dir if @temp_user_data_dir
+        if @pid
+          kill(@pid)
+          kill(@xvfb.pid) if @xvfb&.pid
+          @pid = nil
+        end
+
+        remove_user_data_dir if @user_data_dir
         ObjectSpace.undefine_finalizer(self)
       end
 
@@ -181,14 +117,13 @@ module Ferrum
 
       private
 
-      def kill
-        self.class.process_killer(@pid).call
-        @pid = nil
+      def kill(pid)
+        self.class.process_killer(pid).call
       end
 
-      def remove_temp_user_data_dir
-        self.class.directory_remover(@temp_user_data_dir).call
-        @temp_user_data_dir = nil
+      def remove_user_data_dir
+        self.class.directory_remover(@user_data_dir).call
+        @user_data_dir = nil
       end
 
       def parse_ws_url(read_io, timeout)
@@ -210,8 +145,8 @@ module Ferrum
         end
 
         unless ws_url
-          @logger.puts output if @logger
-          raise "Chrome process did not produce websocket url within #{timeout} seconds"
+          @logger.puts(output) if @logger
+          raise ProcessTimeoutError.new(timeout, output)
         end
       end
 
@@ -221,12 +156,25 @@ module Ferrum
         @port = @ws_url.port
       end
 
+      def parse_browser_versions
+        return unless ws_url.is_a?(Addressable::URI)
+
+        version_url = URI.parse(ws_url.merge(scheme: "http", path: "/json/version"))
+        response = JSON.parse(::Net::HTTP.get(version_url))
+
+        @v8_version = response["V8-Version"]
+        @browser_version = response["Browser"]
+        @webkit_version = response["WebKit-Version"]
+        @default_user_agent = response["User-Agent"]
+        @protocol_version = response["Protocol-Version"]
+      end
+
       def close_io(*ios)
         ios.each do |io|
           begin
             io.close unless io.closed?
           rescue IOError
-            raise unless RUBY_ENGINE == 'jruby'
+            raise unless RUBY_ENGINE == "jruby"
           end
         end
       end