ferrum 0.10.1 → 0.12

data/lib/ferrum/browser/options/chrome.rb

@@ -36,18 +36,27 @@ module Ferrum
           "metrics-recording-only" => nil,
           "safebrowsing-disable-auto-update" => nil,
           "password-store" => "basic",
-          # Note: --no-sandbox is not needed if you properly setup a user in the container.
+          "no-startup-window" => nil
+          # NOTE: --no-sandbox is not needed if you properly setup a user in the container.
           # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
           # "no-sandbox" => nil,
         }.freeze
 
         MAC_BIN_PATH = [
-          "/Applications/Chromium.app/Contents/MacOS/Chromium",
-          "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+          "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+          "/Applications/Chromium.app/Contents/MacOS/Chromium"
         ].freeze
-        LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
-                            google-chrome chrome chromium-browser
-                            google-chrome-stable].freeze
+        LINUX_BIN_PATH = %w[chrome google-chrome google-chrome-stable google-chrome-beta
+                            chromium chromium-browser google-chrome-unstable].freeze
+        WINDOWS_BIN_PATH = [
+          "C:/Program Files/Google/Chrome/Application/chrome.exe",
+          "C:/Program Files/Google/Chrome Dev/Application/chrome.exe"
+        ].freeze
+        PLATFORM_PATH = {
+          mac: MAC_BIN_PATH,
+          windows: WINDOWS_BIN_PATH,
+          linux: LINUX_BIN_PATH
+        }.freeze
 
         def merge_required(flags, options, user_data_dir)
           port = options.fetch(:port, BROWSER_PORT)
@@ -55,14 +64,12 @@ module Ferrum
           flags.merge("remote-debugging-port" => port,
                       "remote-debugging-address" => host,
                       # Doesn't work on MacOS, so we need to set it by CDP
-                      "window-size" => options[:window_size].join(","),
+                      "window-size" => options[:window_size]&.join(","),
                       "user-data-dir" => user_data_dir)
         end
 
         def merge_default(flags, options)
-          unless options.fetch(:headless, true)
-            defaults = except("headless", "disable-gpu")
-          end
+          defaults = except("headless", "disable-gpu") unless options.fetch(:headless, true)
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/options/firefox.rb

@@ -5,13 +5,22 @@ module Ferrum
     module Options
       class Firefox < Base
         DEFAULT_OPTIONS = {
-          "headless" => nil,
+          "headless" => nil
         }.freeze
 
         MAC_BIN_PATH = [
           "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
         ].freeze
         LINUX_BIN_PATH = %w[firefox].freeze
+        WINDOWS_BIN_PATH = [
+          "C:/Program Files/Firefox Developer Edition/firefox.exe",
+          "C:/Program Files/Mozilla Firefox/firefox.exe"
+        ].freeze
+        PLATFORM_PATH = {
+          mac: MAC_BIN_PATH,
+          windows: WINDOWS_BIN_PATH,
+          linux: LINUX_BIN_PATH
+        }.freeze
 
         def merge_required(flags, options, user_data_dir)
           port = options.fetch(:port, BROWSER_PORT)
@@ -21,9 +30,7 @@ module Ferrum
         end
 
         def merge_default(flags, options)
-          unless options.fetch(:headless, true)
-            defaults = except("headless")
-          end
+          defaults = except("headless") unless options.fetch(:headless, true)
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/process.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "cliver"
 require "net/http"
 require "json"
 require "addressable"
@@ -22,7 +21,6 @@ module Ferrum
                   :default_user_agent, :browser_version, :protocol_version,
                   :v8_version, :webkit_version, :xvfb
 
-
       extend Forwardable
       delegate path: :command
 
@@ -32,41 +30,50 @@ module Ferrum
 
       def self.process_killer(pid)
         proc do
-          begin
-            if Ferrum.windows?
+          if Utils::Platform.windows?
+            # Process.kill is unreliable on Windows
+            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
+          else
+            ::Process.kill("USR1", pid)
+            start = Utils::ElapsedTime.monotonic_time
+            while ::Process.wait(pid, ::Process::WNOHANG).nil?
+              sleep(WAIT_KILLED)
+              next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
+
               ::Process.kill("KILL", pid)
-            else
-              ::Process.kill("USR1", pid)
-              start = Ferrum.monotonic_time
-              while ::Process.wait(pid, ::Process::WNOHANG).nil?
-                sleep(WAIT_KILLED)
-                next unless Ferrum.timeout?(start, KILL_TIMEOUT)
-                ::Process.kill("KILL", pid)
-                ::Process.wait(pid)
-                break
-              end
+              ::Process.wait(pid)
+              break
             end
-          rescue Errno::ESRCH, Errno::ECHILD
           end
+        rescue Errno::ESRCH, Errno::ECHILD
+          # nop
         end
       end
 
       def self.directory_remover(path)
-        proc { FileUtils.remove_entry(path) rescue Errno::ENOENT }
+        proc {
+          begin
+            FileUtils.remove_entry(path)
+          rescue StandardError
+            Errno::ENOENT
+          end
+        }
       end
 
       def initialize(options)
+        @pid = @xvfb = @user_data_dir = nil
+
         if options[:url]
           url = URI.join(options[:url].to_s, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
-          set_ws_url(response["webSocketDebuggerUrl"])
+          self.ws_url = response["webSocketDebuggerUrl"]
           parse_browser_versions
           return
         end
 
-        @pid = @xvfb = @user_data_dir = nil
         @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
+        @env = Hash(options[:env])
 
         tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
@@ -81,7 +88,7 @@ module Ferrum
         begin
           read_io, write_io = IO.pipe
           process_options = { in: File::NULL }
-          process_options[:pgroup] = true unless Ferrum.windows?
+          process_options[:pgroup] = true unless Utils::Platform.windows?
           process_options[:out] = process_options[:err] = write_io
 
           if @command.xvfb?
@@ -89,7 +96,8 @@ module Ferrum
             ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
           end
 
-          @pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
+          env = Hash(@xvfb&.to_env).merge(@env)
+          @pid = ::Process.spawn(env, *@command.to_a, process_options)
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
@@ -128,29 +136,29 @@ module Ferrum
 
       def parse_ws_url(read_io, timeout)
         output = ""
-        start = Ferrum.monotonic_time
+        start = Utils::ElapsedTime.monotonic_time
         max_time = start + timeout
-        regexp = /DevTools listening on (ws:\/\/.*)/
-        while (now = Ferrum.monotonic_time) < max_time
+        regexp = %r{DevTools listening on (ws://.*)}
+        while (now = Utils::ElapsedTime.monotonic_time) < max_time
           begin
             output += read_io.read_nonblock(512)
           rescue IO::WaitReadable
-            IO.select([read_io], nil, nil, max_time - now)
+            read_io.wait_readable(max_time - now)
           else
             if output.match(regexp)
-              set_ws_url(output.match(regexp)[1].strip)
+              self.ws_url = output.match(regexp)[1].strip
               break
             end
           end
         end
 
-        unless ws_url
-          @logger.puts(output) if @logger
-          raise ProcessTimeoutError.new(timeout, output)
-        end
+        return if ws_url
+
+        @logger&.puts(output)
+        raise ProcessTimeoutError.new(timeout, output)
       end
 
-      def set_ws_url(url)
+      def ws_url=(url)
         @ws_url = Addressable::URI.parse(url)
         @host = @ws_url.host
         @port = @ws_url.port
@@ -171,11 +179,9 @@ module Ferrum
 
       def close_io(*ios)
         ios.each do |io|
-          begin
-            io.close unless io.closed?
-          rescue IOError
-            raise unless RUBY_ENGINE == "jruby"
-          end
+          io.close unless io.closed?
+        rescue IOError
+          raise unless RUBY_ENGINE == "jruby"
         end
       end
     end

data/lib/ferrum/browser/subscriber.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "concurrent-ruby"
-
 module Ferrum
   class Browser
     class Subscriber
@@ -29,7 +27,7 @@ module Ferrum
         method, params = message.values_at("method", "params")
         total = @on[method].size
         @on[method].each_with_index do |block, index|
-          # If there are a few callback we provide current index and total
+          # In case of multiple callbacks we provide current index and total
           block.call(params, index, total)
         end
       end

data/lib/ferrum/browser/web_socket.rb

@@ -21,9 +21,7 @@ module Ferrum
         @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
         @messages = Queue.new
 
-        if SKIP_LOGGING_SCREENSHOTS
-          @screenshot_commands = Concurrent::Hash.new
-        end
+        @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
 
         @driver.on(:open,    &method(:on_open))
         @driver.on(:message, &method(:on_message))
@@ -31,12 +29,13 @@ module Ferrum
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          if Thread.current.respond_to?(:report_on_exception=)
-            Thread.current.report_on_exception = true
-          end
+          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
 
           begin
-            while data = @sock.readpartial(512)
+            loop do
+              data = @sock.readpartial(512)
+              break unless data
+
               @driver.parse(data)
             end
           rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
@@ -62,7 +61,7 @@ module Ferrum
           output.sub!(/{"data":"(.*)"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
         end
 
-        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{output}\n")
+        @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
       end
 
       def on_close(_event)
@@ -71,13 +70,11 @@ module Ferrum
       end
 
       def send_message(data)
-        if SKIP_LOGGING_SCREENSHOTS
-          @screenshot_commands[data[:id]] = true
-        end
+        @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
         @driver.text(json)
-        @logger&.puts("\n\n▶ #{Ferrum.elapsed_time} #{json}")
+        @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
       def write(data)

data/lib/ferrum/browser/xvfb.rb

@@ -4,23 +4,19 @@ module Ferrum
   class Browser
     class Xvfb
       NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
-                  "it with your package manager".freeze
+                  "it with your package manager"
 
       def self.start(*args)
         new(*args).tap(&:start)
       end
 
-      def self.xvfb_path
-        Cliver.detect("Xvfb")
-      end
-
       attr_reader :screen_size, :display_id, :pid
 
       def initialize(options)
-        @path = self.class.xvfb_path
-        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+        @path = Binary.find("Xvfb")
+        raise BinaryNotFoundError, NOT_FOUND unless @path
 
-        @screen_size = options.fetch(:window_size, [1024, 768]).join("x") + "x24"
+        @screen_size = "#{options.fetch(:window_size, [1024, 768]).join('x')}x24"
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end
 

data/lib/ferrum/browser.rb

@@ -3,10 +3,12 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
+require "ferrum/proxy"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
+require "ferrum/browser/binary"
 
 module Ferrum
   class Browser
@@ -16,21 +18,23 @@ module Ferrum
 
     extend Forwardable
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[go_to back forward refresh reload stop wait_for_reload
+    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
-                body doctype set_content
+                body doctype content=
                 headers cookies network
                 mouse keyboard
                 screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
-                on goto] => :page
+                on position position=
+                playback_rate playback_rate=] => :page
     delegate %i[default_user_agent] => :process
 
     attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
-                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
+                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size, :proxy_options,
+                :proxy_server
     attr_writer :timeout
 
     def initialize(options = nil)
@@ -44,16 +48,29 @@ module Ferrum
       @logger, @timeout, @ws_max_receive_size =
         @options.values_at(:logger, :timeout, :ws_max_receive_size)
       @js_errors = @options.fetch(:js_errors, false)
+
+      if @options[:proxy]
+        @proxy_options = @options[:proxy]
+
+        if @proxy_options[:server]
+          @proxy_server = Proxy.start(**@proxy_options.slice(:host, :port, :user, :password))
+          @proxy_options.merge!(host: @proxy_server.host, port: @proxy_server.port)
+        end
+
+        @options[:browser_options] ||= {}
+        address = "#{@proxy_options[:host]}:#{@proxy_options[:port]}"
+        @options[:browser_options].merge!("proxy-server" => address)
+        @options[:browser_options].merge!("proxy-bypass-list" => @proxy_options[:bypass]) if @proxy_options[:bypass]
+      end
+
       @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
       @slowmo = @options[:slowmo].to_f
 
-      if @options.key?(:base_url)
-        self.base_url = @options[:base_url]
-      end
+      self.base_url = @options[:base_url] if @options.key?(:base_url)
 
-      if ENV["FERRUM_DEBUG"] && !@logger
-        STDOUT.sync = true
-        @logger = STDOUT
+      if ENV.fetch("FERRUM_DEBUG", nil) && !@logger
+        $stdout.sync = true
+        @logger = $stdout
         @options[:logger] = @logger
       end
 
@@ -71,12 +88,32 @@ module Ferrum
       @base_url = parsed
     end
 
+    def create_page(new_context: false)
+      page = if new_context
+               context = contexts.create
+               context.create_page
+             else
+               default_context.create_page
+             end
+
+      block_given? ? yield(page) : page
+    ensure
+      if block_given?
+        page.close
+        context.dispose if new_context
+      end
+    end
+
     def extensions
       @extensions ||= Array(@options[:extensions]).map do |ext|
         (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
       end
     end
 
+    def evaluate_on_new_document(expression)
+      extensions << expression
+    end
+
     def timeout
       @timeout || DEFAULT_TIMEOUT
     end
@@ -116,7 +153,7 @@ module Ferrum
     private
 
     def start
-      Ferrum.started
+      Utils::ElapsedTime.start
       @process = Process.start(@options)
       @client = Client.new(self, @process.ws_url)
       @contexts = Contexts.new(self)

data/lib/ferrum/context.rb

@@ -9,8 +9,10 @@ module Ferrum
     attr_reader :id, :targets
 
     def initialize(browser, contexts, id)
-      @browser, @contexts, @id = browser, contexts, id
-      @targets = Concurrent::Hash.new
+      @id = id
+      @browser = browser
+      @contexts = contexts
+      @targets = Concurrent::Map.new
       @pendings = Concurrent::MVar.new
     end
 
@@ -32,6 +34,7 @@ module Ferrum
     # usually is the last one.
     def windows(pos = nil, size = 1)
       raise ArgumentError if pos && !POSITION.include?(pos)
+
       windows = @targets.values.select(&:window?)
       windows = windows.send(pos, size) if pos
       windows.map(&:page)
@@ -47,14 +50,15 @@ module Ferrum
                        url: "about:blank")
       target = @pendings.take(@browser.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
-      @targets[target.id] = target
+
+      @targets.put_if_absent(target.id, target)
       target
     end
 
     def add_target(params)
       target = Target.new(@browser, params)
       if target.window?
-        @targets[target.id] = target
+        @targets.put_if_absent(target.id, target)
       else
         @pendings.put(target, @browser.timeout)
       end
@@ -72,6 +76,10 @@ module Ferrum
       @contexts.dispose(@id)
     end
 
+    def target?(target_id)
+      !!@targets[target_id]
+    end
+
     def inspect
       %(#<#{self.class} @id=#{@id.inspect} @targets=#{@targets.inspect} @default_target=#{@default_target.inspect}>)
     end

data/lib/ferrum/contexts.rb

@@ -7,7 +7,7 @@ module Ferrum
     attr_reader :contexts
 
     def initialize(browser)
-      @contexts = Concurrent::Hash.new
+      @contexts = Concurrent::Map.new
       @browser = browser
       subscribe
       discover
@@ -18,7 +18,9 @@ module Ferrum
     end
 
     def find_by(target_id:)
-      @contexts.find { |_, c| c.targets.keys.include?(target_id) }&.last
+      context = nil
+      @contexts.each_value { |c| context = c if c.target?(target_id) }
+      context
     end
 
     def create
@@ -39,7 +41,11 @@ module Ferrum
 
     def reset
       @default_context = nil
-      @contexts.keys.each { |id| dispose(id) }
+      @contexts.each_key { |id| dispose(id) }
+    end
+
+    def size
+      @contexts.size
     end
 
     private
@@ -62,15 +68,13 @@ module Ferrum
       end
 
       @browser.client.on("Target.targetDestroyed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
 
       @browser.client.on("Target.targetCrashed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
     end
 

data/lib/ferrum/cookies.rb

@@ -3,6 +3,8 @@
 module Ferrum
   class Cookies
     class Cookie
+      attr_reader :attributes
+
       def initialize(attributes)
         @attributes = attributes
       end
@@ -44,9 +46,7 @@ module Ferrum
       end
 
       def expires
-        if @attributes["expires"] > 0
-          Time.at(@attributes["expires"])
-        end
+        Time.at(@attributes["expires"]) if @attributes["expires"].positive?
       end
     end
 
@@ -56,24 +56,25 @@ module Ferrum
 
     def all
       cookies = @page.command("Network.getAllCookies")["cookies"]
-      cookies.map { |c| [c["name"], Cookie.new(c)] }.to_h
+      cookies.to_h { |c| [c["name"], Cookie.new(c)] }
     end
 
     def [](name)
       all[name]
     end
 
-    def set(name: nil, value: nil, **options)
-      cookie = options.dup
-      cookie[:name]   ||= name
-      cookie[:value]  ||= value
+    def set(options)
+      cookie = (
+        options.is_a?(Cookie) ? options.attributes : options
+      ).dup.transform_keys(&:to_sym)
+
       cookie[:domain] ||= default_domain
 
       cookie[:httpOnly] = cookie.delete(:httponly) if cookie.key?(:httponly)
       cookie[:sameSite] = cookie.delete(:samesite) if cookie.key?(:samesite)
 
       expires = cookie.delete(:expires).to_i
-      cookie[:expires] = expires if expires > 0
+      cookie[:expires] = expires if expires.positive?
 
       @page.command("Network.setCookie", **cookie)["success"]
     end