ferrum 0.11 → 0.13

data/lib/ferrum/page.rb

@@ -10,6 +10,8 @@ require "ferrum/network"
 require "ferrum/page/frames"
 require "ferrum/page/screenshot"
 require "ferrum/page/animation"
+require "ferrum/page/tracing"
+require "ferrum/page/stream"
 require "ferrum/browser/client"
 
 module Ferrum
@@ -32,40 +34,82 @@ module Ferrum
 
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
-                current_url current_title url title body doctype set_content
-                execution_id evaluate evaluate_on evaluate_async execute evaluate_func
+                current_url current_title url title body doctype content=
+                execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
 
-    include Frames, Screenshot, Animation
-
-    attr_accessor :referrer
-    attr_reader :target_id, :browser,
-                :headers, :cookies, :network,
-                :mouse, :keyboard, :event, :document_id
-
-    def initialize(target_id, browser)
-      @frames = {}
+    include Animation
+    include Screenshot
+    include Frames
+    include Stream
+
+    attr_accessor :referrer, :timeout
+    attr_reader :target_id, :browser, :event, :tracing
+
+    # Mouse object.
+    #
+    # @return [Mouse]
+    attr_reader :mouse
+
+    # Keyboard object.
+    #
+    # @return [Keyboard]
+    attr_reader :keyboard
+
+    # Network object.
+    #
+    # @return [Network]
+    attr_reader :network
+
+    # Headers object.
+    #
+    # @return [Headers]
+    attr_reader :headers
+
+    # Cookie store.
+    #
+    # @return [Cookies]
+    attr_reader :cookies
+
+    def initialize(target_id, browser, proxy: nil)
+      @frames = Concurrent::Map.new
       @main_frame = Frame.new(nil, self)
-      @target_id, @browser = target_id, browser
+      @browser = browser
+      @target_id = target_id
+      @timeout = @browser.timeout
       @event = Event.new.tap(&:set)
+      self.proxy = proxy
 
-      host = @browser.process.host
-      port = @browser.process.port
-      ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
-      @client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
+      @client = Browser::Client.new(ws_url, self,
+                                    logger: @browser.options.logger,
+                                    ws_max_receive_size: @browser.options.ws_max_receive_size,
+                                    id_starts_with: 1000)
 
-      @mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
-      @headers, @cookies = Headers.new(self), Cookies.new(self)
+      @mouse = Mouse.new(self)
+      @keyboard = Keyboard.new(self)
+      @headers = Headers.new(self)
+      @cookies = Cookies.new(self)
       @network = Network.new(self)
+      @tracing = Tracing.new(self)
 
       subscribe
       prepare_page
     end
 
-    def timeout
-      @browser.timeout
+    def context
+      @browser.contexts.find_by(target_id: target_id)
     end
 
+    #
+    # Navigates the page to a URL.
+    #
+    # @param [String, nil] url
+    #   The URL to navigate to. The url should include scheme unless you set
+    #   `{Browser#base_url = url}` when configuring driver.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #
     def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
@@ -74,17 +118,20 @@ module Ferrum
       if %w[net::ERR_NAME_NOT_RESOLVED
             net::ERR_NAME_RESOLUTION_FAILED
             net::ERR_INTERNET_DISCONNECTED
-            net::ERR_CONNECTION_TIMED_OUT].include?(response["errorText"])
+            net::ERR_CONNECTION_TIMED_OUT
+            net::ERR_FILE_NOT_FOUND].include?(response["errorText"])
         raise StatusError, options[:url]
       end
+
       response["frameId"]
     rescue TimeoutError
-      if @browser.pending_connection_errors
-        pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
+      if @browser.options.pending_connection_errors
+        pendings = network.traffic.select(&:pending?).map(&:url).compact
         raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
       end
     end
     alias goto go_to
+    alias go go_to
 
     def close
       @headers.clear
@@ -109,27 +156,83 @@ module Ferrum
                                                     fitWindow: false)
     end
 
+    #
+    # The current position of the browser window.
+    #
+    # @return [(Integer, Integer)]
+    #   The left, top coordinates of the browser window.
+    #
+    # @example
+    #   browser.position # => [10, 20]
+    #
     def position
       @browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
     end
 
-    def position=(left:, top:)
-      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: { left: left, top: top })
+    #
+    # Sets the position of the browser window.
+    #
+    # @param [Hash{Symbol => Object}] options
+    #
+    # @option options [Integer] :left
+    #   The number of pixels from the left-hand side of the screen.
+    #
+    # @option options [Integer] :top
+    #   The number of pixels from the top of the screen.
+    #
+    # @example
+    #   browser.position = { left: 10, top: 20 }
+    #
+    def position=(options)
+      @browser.command("Browser.setWindowBounds",
+                       windowId: window_id,
+                       bounds: { left: options[:left], top: options[:top] })
     end
 
+    #
+    # Reloads the current page.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.refresh
+    #
     def refresh
       command("Page.reload", wait: timeout, slowmoable: true)
     end
-    alias_method :reload, :refresh
-
+    alias reload refresh
+
+    #
+    # Stop all navigations and loading pending resources on the page.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.stop
+    #
     def stop
       command("Page.stopLoading", slowmoable: true)
     end
 
+    #
+    # Navigates to the previous URL in the browser's history.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.at_xpath("//a").click
+    #   browser.back
+    #
     def back
       history_navigate(delta: -1)
     end
 
+    #
+    # Navigates to the next URL in the browser's history.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.at_xpath("//a").click
+    #   browser.back
+    #   browser.forward
+    #
     def forward
       history_navigate(delta: 1)
     end
@@ -140,8 +243,21 @@ module Ferrum
       @event.set
     end
 
-    def bypass_csp(value = true)
-      enabled = !!value
+    #
+    # Enables/disables CSP bypass.
+    #
+    # @param [Boolean] enabled
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   browser.bypass_csp # => true
+    #   browser.go_to("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
+    #   browser.refresh
+    #   browser.add_script_tag(content: "window.__injected = 42")
+    #   browser.evaluate("window.__injected") # => 42
+    #
+    def bypass_csp(enabled: true)
       command("Page.setBypassCSP", enabled: enabled)
       enabled
     end
@@ -155,16 +271,17 @@ module Ferrum
     end
 
     def command(method, wait: 0, slowmoable: false, **params)
-      iteration = @event.reset if wait > 0
-      sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
+      iteration = @event.reset if wait.positive?
+      sleep(@browser.options.slowmo) if slowmoable && @browser.options.slowmo.positive?
       result = @client.command(method, params)
 
-      if wait > 0
-        @event.wait(wait) # Wait a bit after command and check if iteration has
-                          # changed which means there was some network event for
-                          # the main frame and it started to load new content.
+      if wait.positive?
+        # Wait a bit after command and check if iteration has
+        # changed which means there was some network event for
+        # the main frame and it started to load new content.
+        @event.wait(wait)
         if iteration != @event.iteration
-          set = @event.wait(@browser.timeout)
+          set = @event.wait(timeout)
           raise TimeoutError unless set
         end
       end
@@ -200,22 +317,41 @@ module Ferrum
       @client.subscribed?(event)
     end
 
+    def use_proxy?
+      @proxy_host && @proxy_port
+    end
+
+    def use_authorized_proxy?
+      use_proxy? && @proxy_user && @proxy_password
+    end
+
     private
 
     def subscribe
       frames_subscribe
       network.subscribe
 
-      if @browser.logger
+      if @browser.options.logger
         on("Runtime.consoleAPICalled") do |params|
-          params["args"].each { |r| @browser.logger.puts(r["value"]) }
+          params["args"].each { |r| @browser.options.logger.puts(r["value"]) }
         end
       end
 
-      if @browser.js_errors
+      if @browser.options.js_errors
         on("Runtime.exceptionThrown") do |params|
-          # FIXME https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
-          Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
+          # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+          Thread.main.raise JavaScriptError.new(
+            params.dig("exceptionDetails", "exception"),
+            params.dig("exceptionDetails", "stackTrace")
+          )
+        end
+      end
+
+      on(:dialog) do |dialog, _index, total|
+        if total == 1
+          warn "Dialog was shown but you didn't provide `on(:dialog)` callback, accepting it by default. " \
+               "Please take a look at https://github.com/rubycdp/ferrum#dialog"
+          dialog.accept
         end
       end
     end
@@ -228,8 +364,23 @@ module Ferrum
       command("Log.enable")
       command("Network.enable")
 
-      if @browser.options[:save_path]
-        command("Page.setDownloadBehavior", behavior: "allow", downloadPath: @browser.options[:save_path])
+      if use_authorized_proxy?
+        network.authorize(user: @proxy_user,
+                          password: @proxy_password,
+                          type: :proxy) do |request, _index, _total|
+          request.continue
+        end
+      end
+
+      if @browser.options.save_path
+        unless Pathname.new(@browser.options.save_path).absolute?
+          raise Error, "supply absolute path for `:save_path` option"
+        end
+
+        @browser.command("Browser.setDownloadBehavior",
+                         browserContextId: context.id,
+                         downloadPath: @browser.options.save_path,
+                         behavior: "allow", eventsEnabled: true)
       end
 
       @browser.extensions.each do |extension|
@@ -242,14 +393,14 @@ module Ferrum
       resize(width: width, height: height)
 
       response = command("Page.getNavigationHistory")
-      if response.dig("entries", 0, "transitionType") != "typed"
-        # If we create page by clicking links, submiting forms and so on it
-        # opens a new window for which `frameStoppedLoading` event never
-        # occurs and thus search for nodes cannot be completed. Here we check
-        # the history and if the transitionType for example `link` then
-        # content is already loaded and we can try to get the document.
-        get_document_id
-      end
+      return unless response.dig("entries", 0, "transitionType") != "typed"
+
+      # If we create page by clicking links, submitting forms and so on it
+      # opens a new window for which `frameStoppedLoading` event never
+      # occurs and thus search for nodes cannot be completed. Here we check
+      # the history and if the transitionType for example `link` then
+      # content is already loaded and we can try to get the document.
+      document_node_id
     end
 
     def inject_extensions
@@ -260,7 +411,7 @@ module Ferrum
         # We also evaluate script just in case because
         # `Page.addScriptToEvaluateOnNewDocument` doesn't work in popups.
         command("Runtime.evaluate", expression: extension,
-                                    contextId: execution_id,
+                                    executionContextId: execution_id!,
                                     returnByValue: true)
       end
     end
@@ -268,13 +419,15 @@ module Ferrum
     def history_navigate(delta:)
       history = command("Page.getNavigationHistory")
       index, entries = history.values_at("currentIndex", "entries")
+      entry = entries[index + delta]
 
-      if entry = entries[index + delta]
-        # Potential wait because of network event
-        command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
-                                               slowmoable: true,
-                                               entryId: entry["id"])
-      end
+      return unless entry
+
+      # Potential wait because of network event
+      command("Page.navigateToHistoryEntry",
+              wait: Mouse::CLICK_WAIT,
+              slowmoable: true,
+              entryId: entry["id"])
     end
 
     def combine_url!(url_or_path)
@@ -288,8 +441,19 @@ module Ferrum
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
     end
 
-    def get_document_id
-      @document_id = command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    def document_node_id
+      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    end
+
+    def ws_url
+      "ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
+    end
+
+    def proxy=(options)
+      @proxy_host = options&.[](:host) || @browser.options.proxy&.[](:host)
+      @proxy_port = options&.[](:port) || @browser.options.proxy&.[](:port)
+      @proxy_user = options&.[](:user) || @browser.options.proxy&.[](:user)
+      @proxy_password = options&.[](:password) || @browser.options.proxy&.[](:password)
     end
   end
 end

data/lib/ferrum/proxy.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "webrick"
+require "webrick/httpproxy"
+
+module Ferrum
+  class Proxy
+    def self.start(**args)
+      new(**args).tap(&:start)
+    end
+
+    attr_reader :host, :port, :user, :password
+
+    def initialize(host: "127.0.0.1", port: 0, user: nil, password: nil)
+      @file = nil
+      @host = host
+      @port = port
+      @user = user
+      @password = password
+    end
+
+    def start
+      options = {
+        ProxyURI: nil, ServerType: Thread,
+        Logger: Logger.new(IO::NULL), AccessLog: [],
+        BindAddress: host, Port: port
+      }
+
+      if user && password
+        @file = Tempfile.new("htpasswd")
+        htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@file.path)
+        htpasswd.set_passwd "Proxy Realm", user, password
+        htpasswd.flush
+        authenticator = WEBrick::HTTPAuth::ProxyBasicAuth.new(Realm: "Proxy Realm",
+                                                              UserDB: htpasswd,
+                                                              Logger: Logger.new(IO::NULL))
+        options.merge!(ProxyAuthProc: authenticator.method(:authenticate).to_proc)
+      end
+
+      @server = HTTPProxyServer.new(**options)
+      @server.start
+      at_exit { stop }
+
+      @port = @server.config[:Port]
+    end
+
+    def rotate(host:, port:, user: nil, password: nil)
+      credentials = "#{user}:#{password}@" if user && password
+      proxy_uri = "schema://#{credentials}#{host}:#{port}"
+      @server.config[:ProxyURI] = URI.parse(proxy_uri)
+    end
+
+    def stop
+      @file&.unlink
+      @server.shutdown
+    end
+
+    # Fix hanging proxy at exit
+    class HTTPProxyServer < WEBrick::HTTPProxyServer
+      # rubocop:disable all
+      def do_CONNECT(req, res)
+        # Proxy Authentication
+        proxy_auth(req, res)
+
+        ua = Thread.current[:WEBrickSocket]  # User-Agent
+        raise WEBrick::HTTPStatus::InternalServerError,
+              "[BUG] cannot get socket" unless ua
+
+        host, port = req.unparsed_uri.split(":", 2)
+        # Proxy authentication for upstream proxy server
+        if proxy = proxy_uri(req, res)
+          proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0"
+          if proxy.userinfo
+            credentials = "Basic " + [proxy.userinfo].pack("m0")
+          end
+          host, port = proxy.host, proxy.port
+        end
+
+        begin
+          @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.")
+          os = TCPSocket.new(host, port)     # origin server
+
+          if proxy
+            @logger.debug("CONNECT: sending a Request-Line")
+            os << proxy_request_line << CRLF
+            @logger.debug("CONNECT: > #{proxy_request_line}")
+            if credentials
+              @logger.debug("CONNECT: sending credentials")
+              os << "Proxy-Authorization: " << credentials << CRLF
+            end
+            os << CRLF
+            proxy_status_line = os.gets(LF)
+            @logger.debug("CONNECT: read Status-Line from the upstream server")
+            @logger.debug("CONNECT: < #{proxy_status_line}")
+            if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line
+              while line = os.gets(LF)
+                break if /\A(#{CRLF}|#{LF})\z/om =~ line
+              end
+            else
+              raise WEBrick::HTTPStatus::BadGateway
+            end
+          end
+          @logger.debug("CONNECT #{host}:#{port}: succeeded")
+          res.status = WEBrick::HTTPStatus::RC_OK
+        rescue => ex
+          @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'")
+          res.set_error(ex)
+          raise WEBrick::HTTPStatus::EOFError
+        ensure
+          # At exit os variable sometimes can be nil which results in hanging forever
+          raise WEBrick::HTTPStatus::EOFError unless os
+
+          if handler = @config[:ProxyContentHandler]
+            handler.call(req, res)
+          end
+          res.send_response(ua)
+          access_log(@config, req, res)
+
+          # Should clear request-line not to send the response twice.
+          # see: HTTPServer#run
+          req.parse(NullReader) rescue nil
+        end
+
+        begin
+          while fds = IO::select([ua, os])
+            if fds[0].member?(ua)
+              buf = ua.readpartial(1024);
+              @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent")
+              os.write(buf)
+            elsif fds[0].member?(os)
+              buf = os.readpartial(1024);
+              @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}")
+              ua.write(buf)
+            end
+          end
+        rescue
+          os.close
+          @logger.debug("CONNECT #{host}:#{port}: closed")
+        end
+
+        raise WEBrick::HTTPStatus::EOFError
+      end
+      # rubocop:enable all
+    end
+  end
+end

data/lib/ferrum/{rbga.rb → rgba.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Ferrum
   class RGBA
     def initialize(red, green, blue, alpha)
@@ -23,13 +25,13 @@ module Ferrum
     end
 
     def validate_color(value)
-      return if value && value.is_a?(Integer) && Range.new(0, 255).include?(value)
+      return if value.is_a?(Integer) && Range.new(0, 255).include?(value)
 
       raise ArgumentError, "Wrong value of #{value} should be Integer from 0 to 255"
     end
 
     def validate_alpha
-      return if alpha && alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
+      return if alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
 
       raise ArgumentError,
             "Wrong alpha value #{alpha} should be Float between 0.0 (fully transparent) and 1.0 (fully opaque)"

data/lib/ferrum/target.rb

@@ -9,6 +9,7 @@ module Ferrum
     attr_writer :page
 
     def initialize(browser, params = nil)
+      @page = nil
       @browser = browser
       @params = params
     end
@@ -22,10 +23,12 @@ module Ferrum
     end
 
     def page
-      @page ||= begin
-        maybe_sleep_if_new_window
-        Page.new(id, @browser)
-      end
+      @page ||= build_page
+    end
+
+    def build_page(**options)
+      maybe_sleep_if_new_window
+      Page.new(id, @browser, **options)
     end
 
     def id

data/lib/ferrum/utils/attempt.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ferrum
+  module Utils
+    module Attempt
+      module_function
+
+      def with_retry(errors:, max:, wait:)
+        attempts ||= 1
+        yield
+      rescue *Array(errors)
+        raise if attempts >= max
+
+        attempts += 1
+        sleep(wait)
+        retry
+      end
+    end
+  end
+end

data/lib/ferrum/utils/elapsed_time.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+
+module Ferrum
+  module Utils
+    module ElapsedTime
+      module_function
+
+      def start
+        @start ||= monotonic_time
+      end
+
+      def elapsed_time(start = nil)
+        monotonic_time - (start || @start)
+      end
+
+      def monotonic_time
+        Concurrent.monotonic_time
+      end
+
+      def timeout?(start, timeout)
+        elapsed_time(start) > timeout
+      end
+    end
+  end
+end

data/lib/ferrum/utils/platform.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Ferrum
+  module Utils
+    module Platform
+      module_function
+
+      def name
+        return :mac if mac?
+        return :windows if windows?
+
+        :linux
+      end
+
+      def windows?
+        RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
+      end
+
+      def mac?
+        RbConfig::CONFIG["host_os"] =~ /darwin/
+      end
+
+      def mri?
+        defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
+      end
+    end
+  end
+end