ferrum 0.11 → 0.13

data/lib/ferrum/browser.rb

@@ -3,89 +3,208 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
+require "ferrum/proxy"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
+require "ferrum/browser/options"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
+require "ferrum/browser/binary"
+require "ferrum/browser/version_info"
 
 module Ferrum
   class Browser
-    DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
-    WINDOW_SIZE = [1024, 768].freeze
-    BASE_URL_SCHEMA = %w[http https].freeze
-
     extend Forwardable
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[go_to back forward refresh reload stop wait_for_reload
+    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
-                body doctype set_content
+                body doctype content=
                 headers cookies network
                 mouse keyboard
                 screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
-                on goto position position=
+                on position position=
                 playback_rate playback_rate=] => :page
     delegate %i[default_user_agent] => :process
 
-    attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
-                :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
-    attr_writer :timeout
+    attr_reader :client, :process, :contexts, :options, :window_size, :base_url
+    attr_accessor :timeout
 
+    #
+    # Initializes the browser.
+    #
+    # @param [Hash{Symbol => Object}, nil] options
+    #   Additional browser options.
+    #
+    # @option options [Boolean] :headless (true)
+    #   Set browser as headless or not.
+    #
+    # @option options [Boolean] :xvfb (false)
+    #   Run browser in a virtual framebuffer.
+    #
+    # @option options [(Integer, Integer)] :window_size ([1024, 768])
+    #   The dimensions of the browser window in which to test, expressed as a
+    #   2-element array, e.g. `[1024, 768]`.
+    #
+    # @option options [Array<String, Hash>] :extensions
+    #   An array of paths to files or JS source code to be preloaded into the
+    #   browser e.g.: `["/path/to/script.js", { source: "window.secret = 'top'" }]`
+    #
+    # @option options [#puts] :logger
+    #   When present, debug output is written to this object.
+    #
+    # @option options [Integer, Float] :slowmo
+    #   Set a delay in seconds to wait before sending command.
+    #   Useful companion of headless option, so that you have time to see
+    #   changes.
+    #
+    # @option options [Numeric] :timeout (5)
+    #   The number of seconds we'll wait for a response when communicating with
+    #   browser.
+    #
+    # @option options [Boolean] :js_errors
+    #   When true, JavaScript errors get re-raised in Ruby.
+    #
+    # @option options [Boolean] :pending_connection_errors (true)
+    #   When main frame is still waiting for slow responses while timeout is
+    #   reached {PendingConnectionsError} is raised. It's better to figure out
+    #   why you have slow responses and fix or block them rather than turn this
+    #   setting off.
+    #
+    # @option options [:chrome, :firefox] :browser_name (:chrome)
+    #   Sets the browser's name. **Note:** only experimental support for
+    #   `:firefox` for now.
+    #
+    # @option options [String] :browser_path
+    #   Path to Chrome binary, you can also set ENV variable as
+    #   `BROWSER_PATH=some/path/chrome bundle exec rspec`.
+    #
+    # @option options [Hash] :browser_options
+    #   Additional command line options, [see them all](https://peter.sh/experiments/chromium-command-line-switches/)
+    #   e.g. `{ "ignore-certificate-errors" => nil }`
+    #
+    # @option options [Boolean] :ignore_default_browser_options
+    #   Ferrum has a number of default options it passes to the browser,
+    #   if you set this to `true` then only options you put in
+    #   `:browser_options` will be passed to the browser, except required ones
+    #   of course.
+    #
+    # @option options [Integer] :port
+    #   Remote debugging port for headless Chrome.
+    #
+    # @option options [String] :host
+    #   Remote debugging address for headless Chrome.
+    #
+    # @option options [String] :url
+    #   URL for a running instance of Chrome. If this is set, a browser process
+    #   will not be spawned.
+    #
+    # @option options [Integer] :process_timeout
+    #   How long to wait for the Chrome process to respond on startup.
+    #
+    # @option options [Integer] :ws_max_receive_size
+    #   How big messages to accept from Chrome over the web socket, in bytes.
+    #   Defaults to 64MB. Incoming messages larger this will cause a
+    #   {Ferrum::DeadBrowserError}.
+    #
+    # @option options [Hash] :proxy
+    #   Specify proxy settings, [read more](https://github.com/rubycdp/ferrum#proxy).
+    #
+    # @option options [String] :save_path
+    #   Path to save attachments with [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition)
+    #   header.
+    #
+    # @option options [Hash] :env
+    #   Environment variables you'd like to pass through to the process.
+    #
     def initialize(options = nil)
-      options ||= {}
+      @options = Options.new(options)
+      @client = @process = @contexts = nil
 
-      @client = nil
-      @window_size = options.fetch(:window_size, WINDOW_SIZE)
-      @original_window_size = @window_size
+      @timeout = @options.timeout
+      @window_size = @options.window_size
+      @base_url = @options.base_url if @options.base_url
 
-      @options = Hash(options.merge(window_size: @window_size))
-      @logger, @timeout, @ws_max_receive_size =
-        @options.values_at(:logger, :timeout, :ws_max_receive_size)
-      @js_errors = @options.fetch(:js_errors, false)
-      @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
-      @slowmo = @options[:slowmo].to_f
+      start
+    end
 
-      if @options.key?(:base_url)
-        self.base_url = @options[:base_url]
-      end
+    #
+    # Sets the base URL.
+    #
+    # @param [String] value
+    #   The new base URL value.
+    #
+    # @raise [ArgumentError] when path is not absolute or doesn't include schema
+    #
+    # @return [Addressable::URI]
+    #   The parsed base URI value.
+    #
+    def base_url=(value)
+      @base_url = options.parse_base_url(value)
+    end
 
-      if ENV["FERRUM_DEBUG"] && !@logger
-        STDOUT.sync = true
-        @logger = STDOUT
-        @options[:logger] = @logger
-      end
+    #
+    # Creates a new page.
+    #
+    # @param [Boolean] new_context
+    #   Whether to create a page in a new context or not.
+    #
+    # @param [Hash] proxy
+    #   Whether to use proxy for a page. The page will be created in a new context if so.
+    #
+    # @return [Ferrum::Page]
+    #   Created page.
+    #
+    def create_page(new_context: false, proxy: nil)
+      page = if new_context || proxy
+               params = {}
 
-      @options.freeze
+               if proxy
+                 options.parse_proxy(proxy)
+                 params.merge!(proxyServer: "#{proxy[:host]}:#{proxy[:port]}")
+                 params.merge!(proxyBypassList: proxy[:bypass]) if proxy[:bypass]
+               end
 
-      start
-    end
+               context = contexts.create(**params)
+               context.create_page(proxy: proxy)
+             else
+               default_context.create_page
+             end
 
-    def base_url=(value)
-      parsed = Addressable::URI.parse(value)
-      unless BASE_URL_SCHEMA.include?(parsed.normalized_scheme)
-        raise "Set `base_url` should be absolute and include schema: #{BASE_URL_SCHEMA}"
+      block_given? ? yield(page) : page
+    ensure
+      if block_given?
+        page.close
+        context.dispose if new_context
       end
-
-      @base_url = parsed
     end
 
     def extensions
-      @extensions ||= Array(@options[:extensions]).map do |ext|
+      @extensions ||= Array(options.extensions).map do |ext|
         (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
       end
     end
 
+    #
+    # Evaluate JavaScript to modify things before a page load.
+    #
+    # @param [String] expression
+    #   The JavaScript to add to each new document.
+    #
+    # @example
+    #   browser.evaluate_on_new_document <<~JS
+    #     Object.defineProperty(navigator, "languages", {
+    #       get: function() { return ["tlh"]; }
+    #     });
+    #   JS
+    #
     def evaluate_on_new_document(expression)
       extensions << expression
     end
 
-    def timeout
-      @timeout || DEFAULT_TIMEOUT
-    end
-
     def command(*args)
       @client.command(*args)
     rescue DeadBrowserError
@@ -93,8 +212,22 @@ module Ferrum
       raise
     end
 
+    #
+    # Closes browser tabs opened by the `Browser` instance.
+    #
+    # @example
+    #   # connect to a long-running Chrome process
+    #   browser = Ferrum::Browser.new(url: 'http://localhost:9222')
+    #
+    #   browser.go_to("https://github.com/")
+    #
+    #   # clean up, lest the tab stays there hanging forever
+    #   browser.reset
+    #
+    #   browser.quit
+    #
     def reset
-      @window_size = @original_window_size
+      @window_size = options.window_size
       contexts.reset
     end
 
@@ -118,12 +251,25 @@ module Ferrum
       command("Browser.crash")
     end
 
+    #
+    # Gets the version information from the browser.
+    #
+    # @return [VersionInfo]
+    #
+    # @since 0.13
+    #
+    def version
+      VersionInfo.new(command("Browser.getVersion"))
+    end
+
     private
 
     def start
-      Ferrum.started
-      @process = Process.start(@options)
-      @client = Client.new(self, @process.ws_url)
+      Utils::ElapsedTime.start
+      @process = Process.start(options)
+      @client = Client.new(@process.ws_url, self,
+                           logger: options.logger,
+                           ws_max_receive_size: options.ws_max_receive_size)
       @contexts = Contexts.new(self)
     end
   end

data/lib/ferrum/context.rb

@@ -9,7 +9,9 @@ module Ferrum
     attr_reader :id, :targets
 
     def initialize(browser, contexts, id)
-      @browser, @contexts, @id = browser, contexts, id
+      @id = id
+      @browser = browser
+      @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::MVar.new
     end
@@ -32,13 +34,15 @@ module Ferrum
     # usually is the last one.
     def windows(pos = nil, size = 1)
       raise ArgumentError if pos && !POSITION.include?(pos)
+
       windows = @targets.values.select(&:window?)
       windows = windows.send(pos, size) if pos
       windows.map(&:page)
     end
 
-    def create_page
-      create_target.page
+    def create_page(**options)
+      target = create_target
+      target.page = target.build_page(**options)
     end
 
     def create_target
@@ -47,6 +51,7 @@ module Ferrum
                        url: "about:blank")
       target = @pendings.take(@browser.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
+
       @targets.put_if_absent(target.id, target)
       target
     end
@@ -72,7 +77,7 @@ module Ferrum
       @contexts.dispose(@id)
     end
 
-    def has_target?(target_id)
+    def target?(target_id)
       !!@targets[target_id]
     end
 

data/lib/ferrum/contexts.rb

@@ -19,12 +19,12 @@ module Ferrum
 
     def find_by(target_id:)
       context = nil
-      @contexts.each_value { |c| context = c if c.has_target?(target_id) }
+      @contexts.each_value { |c| context = c if c.target?(target_id) }
       context
     end
 
-    def create
-      response = @browser.command("Target.createBrowserContext")
+    def create(**options)
+      response = @browser.command("Target.createBrowserContext", **options)
       context_id = response["browserContextId"]
       context = Context.new(@browser, self, context_id)
       @contexts[context_id] = context
@@ -41,7 +41,11 @@ module Ferrum
 
     def reset
       @default_context = nil
-      @contexts.keys.each { |id| dispose(id) }
+      @contexts.each_key { |id| dispose(id) }
+    end
+
+    def size
+      @contexts.size
     end
 
     private
@@ -64,15 +68,13 @@ module Ferrum
       end
 
       @browser.client.on("Target.targetDestroyed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
 
       @browser.client.on("Target.targetCrashed") do |params|
-        if context = find_by(target_id: params["targetId"])
-          context.delete_target(params["targetId"])
-        end
+        context = find_by(target_id: params["targetId"])
+        context&.delete_target(params["targetId"])
       end
     end
 

data/lib/ferrum/cookies/cookie.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Cookies
+    #
+    # Represents a [cookie value](https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-Cookie).
+    #
+    class Cookie
+      # The parsed JSON attributes.
+      #
+      # @return [Hash{String => [String, Boolean, nil]}]
+      attr_reader :attributes
+
+      #
+      # Initializes the cookie.
+      #
+      # @param [Hash{String => String}] attributes
+      #   The parsed JSON attributes.
+      #
+      def initialize(attributes)
+        @attributes = attributes
+      end
+
+      #
+      # The cookie's name.
+      #
+      # @return [String]
+      #
+      def name
+        attributes["name"]
+      end
+
+      #
+      # The cookie's value.
+      #
+      # @return [String]
+      #
+      def value
+        attributes["value"]
+      end
+
+      #
+      # The cookie's domain.
+      #
+      # @return [String]
+      #
+      def domain
+        attributes["domain"]
+      end
+
+      #
+      # The cookie's path.
+      #
+      # @return [String]
+      #
+      def path
+        attributes["path"]
+      end
+
+      #
+      # The `sameSite` configuration.
+      #
+      # @return ["Strict", "Lax", "None", nil]
+      #
+      def samesite
+        attributes["sameSite"]
+      end
+      alias same_site samesite
+
+      #
+      # The cookie's size.
+      #
+      # @return [Integer]
+      #
+      def size
+        attributes["size"]
+      end
+
+      #
+      # Specifies whether the cookie is secure or not.
+      #
+      # @return [Boolean]
+      #
+      def secure?
+        attributes["secure"]
+      end
+
+      #
+      # Specifies whether the cookie is HTTP-only or not.
+      #
+      # @return [Boolean]
+      #
+      def httponly?
+        attributes["httpOnly"]
+      end
+      alias http_only? httponly?
+
+      #
+      # Specifies whether the cookie is a session cookie or not.
+      #
+      # @return [Boolean]
+      #
+      def session?
+        attributes["session"]
+      end
+
+      #
+      # Specifies when the cookie will expire.
+      #
+      # @return [Time, nil]
+      #
+      def expires
+        Time.at(attributes["expires"]) if attributes["expires"].positive?
+      end
+
+      #
+      # Compares different cookie objects.
+      #
+      # @return [Boolean]
+      #
+      def ==(other)
+        other.class == self.class && other.attributes == attributes
+      end
+    end
+  end
+end

data/lib/ferrum/cookies.rb

@@ -1,84 +1,114 @@
 # frozen_string_literal: true
 
+require "ferrum/cookies/cookie"
+
 module Ferrum
   class Cookies
-    class Cookie
-      def initialize(attributes)
-        @attributes = attributes
-      end
-
-      def name
-        @attributes["name"]
-      end
-
-      def value
-        @attributes["value"]
-      end
-
-      def domain
-        @attributes["domain"]
-      end
-
-      def path
-        @attributes["path"]
-      end
-
-      def samesite
-        @attributes["sameSite"]
-      end
-
-      def size
-        @attributes["size"]
-      end
-
-      def secure?
-        @attributes["secure"]
-      end
-
-      def httponly?
-        @attributes["httpOnly"]
-      end
-
-      def session?
-        @attributes["session"]
-      end
-
-      def expires
-        if @attributes["expires"] > 0
-          Time.at(@attributes["expires"])
-        end
-      end
-    end
-
     def initialize(page)
       @page = page
     end
 
+    #
+    # Returns cookies hash.
+    #
+    # @return [Hash{String => Cookie}]
+    #
+    # @example
+    #   browser.cookies.all # => {
+    #   #  "NID" => #<Ferrum::Cookies::Cookie:0x0000558624b37a40 @attributes={
+    #   #     "name"=>"NID", "value"=>"...", "domain"=>".google.com", "path"=>"/",
+    #   #     "expires"=>1583211046.575681, "size"=>178, "httpOnly"=>true, "secure"=>false, "session"=>false
+    #   #  }>
+    #   # }
+    #
     def all
       cookies = @page.command("Network.getAllCookies")["cookies"]
-      cookies.map { |c| [c["name"], Cookie.new(c)] }.to_h
+      cookies.to_h { |c| [c["name"], Cookie.new(c)] }
     end
 
+    #
+    # Returns cookie.
+    #
+    # @param [String] name
+    #   The cookie name to fetch.
+    #
+    # @return [Cookie, nil]
+    #   The cookie with the matching name.
+    #
+    # @example
+    #   browser.cookies["NID"] # =>
+    #   # <Ferrum::Cookies::Cookie:0x0000558624b67a88 @attributes={
+    #   #  "name"=>"NID", "value"=>"...", "domain"=>".google.com",
+    #   #  "path"=>"/", "expires"=>1583211046.575681, "size"=>178,
+    #   #  "httpOnly"=>true, "secure"=>false, "session"=>false
+    #   # }>
+    #
     def [](name)
       all[name]
     end
 
-    def set(name: nil, value: nil, **options)
-      cookie = options.dup
-      cookie[:name]   ||= name
-      cookie[:value]  ||= value
+    #
+    # Sets a cookie.
+    #
+    # @param [Hash{Symbol => Object}, Cookie] options
+    #
+    # @option options [String] :name
+    #
+    # @option options [String] :value
+    #
+    # @option options [String] :domain
+    #
+    # @option options [String] :path
+    #
+    # @option options [Integer] :expires
+    #
+    # @option options [Integer] :size
+    #
+    # @option options [Boolean] :httponly
+    #
+    # @option options [Boolean] :secure
+    #
+    # @option options [String] :samesite
+    #
+    #
+    # @example
+    #   browser.cookies.set(name: "stealth", value: "omg", domain: "google.com") # => true
+    #
+    # @example
+    #   nid_cookie = browser.cookies["NID"] # => <Ferrum::Cookies::Cookie:0x0000558624b67a88>
+    #   browser.cookies.set(nid_cookie) # => true
+    #
+    def set(options)
+      cookie = (
+        options.is_a?(Cookie) ? options.attributes : options
+      ).dup.transform_keys(&:to_sym)
+
       cookie[:domain] ||= default_domain
 
       cookie[:httpOnly] = cookie.delete(:httponly) if cookie.key?(:httponly)
       cookie[:sameSite] = cookie.delete(:samesite) if cookie.key?(:samesite)
 
       expires = cookie.delete(:expires).to_i
-      cookie[:expires] = expires if expires > 0
+      cookie[:expires] = expires if expires.positive?
 
       @page.command("Network.setCookie", **cookie)["success"]
     end
 
-    # Supports :url, :domain and :path options
+    #
+    # Removes given cookie.
+    #
+    # @param [String] name
+    #
+    # @param [Hash{Symbol => Object}] options
+    #   Additional keyword arguments.
+    #
+    # @option options [String] :domain
+    #
+    # @option options [String] :url
+    #
+    # @example
+    #   browser.cookies.remove(name: "stealth", domain: "google.com") # => true
+    #
     def remove(name:, **options)
       raise "Specify :domain or :url option" if !options[:domain] && !options[:url] && !default_domain
 
@@ -90,6 +120,14 @@ module Ferrum
       true
     end
 
+    #
+    # Removes all cookies for current page.
+    #
+    # @return [true]
+    #
+    # @example
+    #   browser.cookies.clear # => true
+    #
     def clear
       @page.command("Network.clearBrowserCookies")
       true