ferrum 0.11 → 0.12

data/lib/ferrum/browser/process.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "cliver"
 require "net/http"
 require "json"
 require "addressable"
@@ -22,7 +21,6 @@ module Ferrum
                   :default_user_agent, :browser_version, :protocol_version,
                   :v8_version, :webkit_version, :xvfb
 
-
       extend Forwardable
       delegate path: :command
 
@@ -32,41 +30,50 @@ module Ferrum
 
       def self.process_killer(pid)
         proc do
-          begin
-            if Ferrum.windows?
+          if Utils::Platform.windows?
+            # Process.kill is unreliable on Windows
+            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
+          else
+            ::Process.kill("USR1", pid)
+            start = Utils::ElapsedTime.monotonic_time
+            while ::Process.wait(pid, ::Process::WNOHANG).nil?
+              sleep(WAIT_KILLED)
+              next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
+
               ::Process.kill("KILL", pid)
-            else
-              ::Process.kill("USR1", pid)
-              start = Ferrum.monotonic_time
-              while ::Process.wait(pid, ::Process::WNOHANG).nil?
-                sleep(WAIT_KILLED)
-                next unless Ferrum.timeout?(start, KILL_TIMEOUT)
-                ::Process.kill("KILL", pid)
-                ::Process.wait(pid)
-                break
-              end
+              ::Process.wait(pid)
+              break
             end
-          rescue Errno::ESRCH, Errno::ECHILD
           end
+        rescue Errno::ESRCH, Errno::ECHILD
+          # nop
         end
       end
 
       def self.directory_remover(path)
-        proc { FileUtils.remove_entry(path) rescue Errno::ENOENT }
+        proc {
+          begin
+            FileUtils.remove_entry(path)
+          rescue StandardError
+            Errno::ENOENT
+          end
+        }
       end
 
       def initialize(options)
+        @pid = @xvfb = @user_data_dir = nil
+
         if options[:url]
           url = URI.join(options[:url].to_s, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
-          set_ws_url(response["webSocketDebuggerUrl"])
+          self.ws_url = response["webSocketDebuggerUrl"]
           parse_browser_versions
           return
         end
 
-        @pid = @xvfb = @user_data_dir = nil
         @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
+        @env = Hash(options[:env])
 
         tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
@@ -81,7 +88,7 @@ module Ferrum
         begin
           read_io, write_io = IO.pipe
           process_options = { in: File::NULL }
-          process_options[:pgroup] = true unless Ferrum.windows?
+          process_options[:pgroup] = true unless Utils::Platform.windows?
           process_options[:out] = process_options[:err] = write_io
 
           if @command.xvfb?
@@ -89,7 +96,8 @@ module Ferrum
             ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
           end
 
-          @pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
+          env = Hash(@xvfb&.to_env).merge(@env)
+          @pid = ::Process.spawn(env, *@command.to_a, process_options)
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
@@ -128,29 +136,29 @@ module Ferrum
 
       def parse_ws_url(read_io, timeout)
         output = ""
-        start = Ferrum.monotonic_time
+        start = Utils::ElapsedTime.monotonic_time
         max_time = start + timeout
-        regexp = /DevTools listening on (ws:\/\/.*)/
-        while (now = Ferrum.monotonic_time) < max_time
+        regexp = %r{DevTools listening on (ws://.*)}
+        while (now = Utils::ElapsedTime.monotonic_time) < max_time
           begin
             output += read_io.read_nonblock(512)
           rescue IO::WaitReadable
-            IO.select([read_io], nil, nil, max_time - now)
+            read_io.wait_readable(max_time - now)
           else
             if output.match(regexp)
-              set_ws_url(output.match(regexp)[1].strip)
+              self.ws_url = output.match(regexp)[1].strip
               break
             end
           end
         end
 
-        unless ws_url
-          @logger.puts(output) if @logger
-          raise ProcessTimeoutError.new(timeout, output)
-        end
+        return if ws_url
+
+        @logger&.puts(output)
+        raise ProcessTimeoutError.new(timeout, output)
       end
 
-      def set_ws_url(url)
+      def ws_url=(url)
         @ws_url = Addressable::URI.parse(url)
         @host = @ws_url.host
         @port = @ws_url.port
@@ -171,11 +179,9 @@ module Ferrum
 
       def close_io(*ios)
         ios.each do |io|
-          begin
-            io.close unless io.closed?
-          rescue IOError
-            raise unless RUBY_ENGINE == "jruby"
-          end
+          io.close unless io.closed?
+        rescue IOError
+          raise unless RUBY_ENGINE == "jruby"
         end
       end
     end

data/lib/ferrum/browser/subscriber.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "concurrent-ruby"
-
 module Ferrum
   class Browser
     class Subscriber
@@ -29,7 +27,7 @@ module Ferrum
         method, params = message.values_at("method", "params")
         total = @on[method].size
         @on[method].each_with_index do |block, index|
-          # If there are a few callback we provide current index and total
+          # In case of multiple callbacks we provide current index and total
           block.call(params, index, total)
         end
       end

data/lib/ferrum/browser/web_socket.rb

@@ -21,9 +21,7 @@ module Ferrum
         @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
         @messages = Queue.new
 
-        if SKIP_LOGGING_SCREENSHOTS
-          @screenshot_commands = Concurrent::Hash.new
-        end
+        @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
 
         @driver.on(:open,    &method(:on_open))
         @driver.on(:message, &method(:on_message))
@@ -31,12 +29,13 @@ module Ferrum
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          if Thread.current.respond_to?(:report_on_exception=)
-            Thread.current.report_on_exception = true
-          end
+          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
 
           begin
-            while data = @sock.readpartial(512)
+            loop do
+              data = @sock.readpartial(512)
+              break unless data
+
               @driver.parse(data)
             end
           rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
@@ -62,7 +61,7 @@ module Ferrum
           output.sub!(/{"data":"(.*)"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
         end
 
-        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{output}\n")
+        @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
       end
 
       def on_close(_event)
@@ -71,13 +70,11 @@ module Ferrum
       end
 
       def send_message(data)
-        if SKIP_LOGGING_SCREENSHOTS
-          @screenshot_commands[data[:id]] = true
-        end
+        @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
         @driver.text(json)
-        @logger&.puts("\n\n▶ #{Ferrum.elapsed_time} #{json}")
+        @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
       def write(data)

data/lib/ferrum/browser/xvfb.rb

@@ -4,23 +4,19 @@ module Ferrum
   class Browser
     class Xvfb
       NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
-                  "it with your package manager".freeze
+                  "it with your package manager"
 
       def self.start(*args)
         new(*args).tap(&:start)
       end
 
-      def self.xvfb_path
-        Cliver.detect("Xvfb")
-      end
-
       attr_reader :screen_size, :display_id, :pid
 
       def initialize(options)
-        @path = self.class.xvfb_path
-        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+        @path = Binary.find("Xvfb")
+        raise BinaryNotFoundError, NOT_FOUND unless @path
 
-        @screen_size = options.fetch(:window_size, [1024, 768]).join("x") + "x24"
+        @screen_size = "#{options.fetch(:window_size, [1024, 768]).join('x')}x24"
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end
 

data/lib/ferrum/browser.rb

@@ -3,10 +3,12 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
+require "ferrum/proxy"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
+require "ferrum/browser/binary"
 
 module Ferrum
   class Browser
@@ -16,22 +18,23 @@ module Ferrum
 
     extend Forwardable
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[go_to back forward refresh reload stop wait_for_reload
+    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
-                body doctype set_content
+                body doctype content=
                 headers cookies network
                 mouse keyboard
                 screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
-                on goto position position=
+                on position position=
                 playback_rate playback_rate=] => :page
     delegate %i[default_user_agent] => :process
 
     attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
-                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
+                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size, :proxy_options,
+                :proxy_server
     attr_writer :timeout
 
     def initialize(options = nil)
@@ -45,16 +48,29 @@ module Ferrum
       @logger, @timeout, @ws_max_receive_size =
         @options.values_at(:logger, :timeout, :ws_max_receive_size)
       @js_errors = @options.fetch(:js_errors, false)
+
+      if @options[:proxy]
+        @proxy_options = @options[:proxy]
+
+        if @proxy_options[:server]
+          @proxy_server = Proxy.start(**@proxy_options.slice(:host, :port, :user, :password))
+          @proxy_options.merge!(host: @proxy_server.host, port: @proxy_server.port)
+        end
+
+        @options[:browser_options] ||= {}
+        address = "#{@proxy_options[:host]}:#{@proxy_options[:port]}"
+        @options[:browser_options].merge!("proxy-server" => address)
+        @options[:browser_options].merge!("proxy-bypass-list" => @proxy_options[:bypass]) if @proxy_options[:bypass]
+      end
+
       @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
       @slowmo = @options[:slowmo].to_f
 
-      if @options.key?(:base_url)
-        self.base_url = @options[:base_url]
-      end
+      self.base_url = @options[:base_url] if @options.key?(:base_url)
 
-      if ENV["FERRUM_DEBUG"] && !@logger
-        STDOUT.sync = true
-        @logger = STDOUT
+      if ENV.fetch("FERRUM_DEBUG", nil) && !@logger
+        $stdout.sync = true
+        @logger = $stdout
         @options[:logger] = @logger
       end
 
@@ -72,6 +88,22 @@ module Ferrum
       @base_url = parsed
     end
 
+    def create_page(new_context: false)
+      page = if new_context
+               context = contexts.create
+               context.create_page
+             else
+               default_context.create_page
+             end
+
+      block_given? ? yield(page) : page
+    ensure
+      if block_given?
+        page.close
+        context.dispose if new_context
+      end
+    end
+
     def extensions
       @extensions ||= Array(@options[:extensions]).map do |ext|
         (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
@@ -121,7 +153,7 @@ module Ferrum
     private
 
     def start
-      Ferrum.started
+      Utils::ElapsedTime.start
       @process = Process.start(@options)
       @client = Client.new(self, @process.ws_url)
       @contexts = Contexts.new(self)

data/lib/ferrum/context.rb

@@ -9,7 +9,9 @@ module Ferrum
     attr_reader :id, :targets
 
     def initialize(browser, contexts, id)
-      @browser, @contexts, @id = browser, contexts, id
+      @id = id
+      @browser = browser
+      @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::MVar.new
     end
@@ -32,6 +34,7 @@ module Ferrum
     # usually is the last one.
     def windows(pos = nil, size = 1)
       raise ArgumentError if pos && !POSITION.include?(pos)
+
       windows = @targets.values.select(&:window?)
       windows = windows.send(pos, size) if pos
       windows.map(&:page)
@@ -47,6 +50,7 @@ module Ferrum
                        url: "about:blank")
       target = @pendings.take(@browser.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
+
       @targets.put_if_absent(target.id, target)
       target
     end
@@ -72,7 +76,7 @@ module Ferrum
       @contexts.dispose(@id)
     end
 
-    def has_target?(target_id)
+    def target?(target_id)
       !!@targets[target_id]
     end
 

data/lib/ferrum/contexts.rb

@@ -19,7 +19,7 @@ module Ferrum
 
     def find_by(target_id:)
       context = nil
-      @contexts.each_value { |c| context = c if c.has_target?(target_id) }
+      @contexts.each_value { |c| context = c if c.target?(target_id) }
       context
     end
 
@@ -41,7 +41,11 @@ module Ferrum
 
     def reset
       @default_context = nil
-      @contexts.keys.each { |id| dispose(id) }
+      @contexts.each_key { |id| dispose(id) }
+    end
+
+    def size
+      @contexts.size
     end
 
     private
@@ -64,15 +68,13 @@ module Ferrum
       end
 
       @browser.client.on("Target.targetDestroyed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
 
       @browser.client.on("Target.targetCrashed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
     end
 

data/lib/ferrum/cookies.rb

@@ -3,6 +3,8 @@
 module Ferrum
   class Cookies
     class Cookie
+      attr_reader :attributes
+
       def initialize(attributes)
         @attributes = attributes
       end
@@ -44,9 +46,7 @@ module Ferrum
       end
 
       def expires
-        if @attributes["expires"] > 0
-          Time.at(@attributes["expires"])
-        end
+        Time.at(@attributes["expires"]) if @attributes["expires"].positive?
       end
     end
 
@@ -56,24 +56,25 @@ module Ferrum
 
     def all
       cookies = @page.command("Network.getAllCookies")["cookies"]
-      cookies.map { |c| [c["name"], Cookie.new(c)] }.to_h
+      cookies.to_h { |c| [c["name"], Cookie.new(c)] }
     end
 
     def [](name)
       all[name]
     end
 
-    def set(name: nil, value: nil, **options)
-      cookie = options.dup
-      cookie[:name]   ||= name
-      cookie[:value]  ||= value
+    def set(options)
+      cookie = (
+        options.is_a?(Cookie) ? options.attributes : options
+      ).dup.transform_keys(&:to_sym)
+
       cookie[:domain] ||= default_domain
 
       cookie[:httpOnly] = cookie.delete(:httponly) if cookie.key?(:httponly)
       cookie[:sameSite] = cookie.delete(:samesite) if cookie.key?(:samesite)
 
       expires = cookie.delete(:expires).to_i
-      cookie[:expires] = expires if expires > 0
+      cookie[:expires] = expires if expires.positive?
 
       @page.command("Network.setCookie", **cookie)["success"]
     end

data/lib/ferrum/errors.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Error               < StandardError; end
+  class NoSuchPageError     < Error; end
+  class NoSuchTargetError   < Error; end
+  class NotImplementedError < Error; end
+  class BinaryNotFoundError < Error; end
+  class EmptyPathError < Error; end
+
+  class StatusError < Error
+    def initialize(url, message = nil)
+      super(message || "Request to #{url} failed to reach server, check DNS and server status")
+    end
+  end
+
+  class PendingConnectionsError < StatusError
+    attr_reader :pendings
+
+    def initialize(url, pendings = [])
+      @pendings = pendings
+
+      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+
+      super(url, message)
+    end
+  end
+
+  class TimeoutError < Error
+    def message
+      "Timed out waiting for response. It's possible that this happened " \
+        "because something took a very long time (for example a page load " \
+        "was slow). If so, setting the :timeout option to a higher value might " \
+        "help."
+    end
+  end
+
+  class ScriptTimeoutError < Error
+    def message
+      "Timed out waiting for evaluated script to return a value"
+    end
+  end
+
+  class ProcessTimeoutError < Error
+    attr_reader :output
+
+    def initialize(timeout, output)
+      @output = output
+      super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
+    end
+  end
+
+  class DeadBrowserError < Error
+    def initialize(message = "Browser is dead or given window is closed")
+      super
+    end
+  end
+
+  class NodeMovingError < Error
+    def initialize(node, prev, current)
+      @node = node
+      @prev = prev
+      @current = current
+      super(message)
+    end
+
+    def message
+      "#{@node.inspect} that you're trying to click is moving, hence " \
+        "we cannot. Previously it was at #{@prev.inspect} but now at " \
+        "#{@current.inspect}."
+    end
+  end
+
+  class CoordinatesNotFoundError < Error
+    def initialize(message = "Could not compute content quads")
+      super
+    end
+  end
+
+  class BrowserError < Error
+    attr_reader :response
+
+    def initialize(response)
+      @response = response
+      super(response["message"])
+    end
+
+    def code
+      response["code"]
+    end
+
+    def data
+      response["data"]
+    end
+  end
+
+  class NodeNotFoundError < BrowserError; end
+
+  class NoExecutionContextError < BrowserError
+    def initialize(response = nil)
+      response ||= { "message" => "There's no context available" }
+      super(response)
+    end
+  end
+
+  class JavaScriptError < BrowserError
+    attr_reader :class_name, :message, :stack_trace
+
+    def initialize(response, stack_trace = nil)
+      @class_name, @message = response.values_at("className", "description")
+      @stack_trace = stack_trace
+      super(response.merge("message" => @message))
+    end
+  end
+end