ferrum 0.12 → 0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e4ca57662283c8c6b917aa3e2f1f06b154cb66ffdd8522756f55e09be84a423
-  data.tar.gz: 50d37d615399fe481634154d619d55872ee13b0cace511a1b147da38454247b5
+  metadata.gz: bde7c8e40700ace2d713cba69eee5828dcb888e5468c07a6b1e5e0d668e4c641
+  data.tar.gz: d52f7278dd76e670aa50721e6d70adc26297bbfb031ddac259dde5421c817a04
 SHA512:
-  metadata.gz: 9cc514740ae8041b61a184bdeee0a44ce26b276f36af0358c6a00524f8e13b6db9ddda48acf5781c4122acaa4980579cf06053ffb420c2013e0b44177eab2571
-  data.tar.gz: cc00b6d3cb440db600241156354dcd92a6bf67c0402b9e2881f32ea507549d80f69b51011bb246baf9faf95f33e5596933ce569cd9b03f41c7cc066d215b6073
+  metadata.gz: a7206d7a92d8483bd106fe262130492e7bf51e2db8b99f73c39ada0a674fb29c84bc948bdbb6f554b672ade9f4e3812a9158447b30d6f976cb4892b5e4e8df30
+  data.tar.gz: aef76b65c27dca2a5385d9881f9be8caccb85e804b31a26e421e35a43295baa3a5c818de856b49e3a7928139af4aeb588a5fae1cb67275440e5a0b3ddf97f76e

data/README.md

@@ -154,7 +154,7 @@ Ferrum::Browser.new(options)
   * `:logger` (Object responding to `puts`) - When present, debug output is
       written to this object.
   * `:slowmo` (Integer | Float) - Set a delay in seconds to wait before sending command.
-      Usefull companion of headless option, so that you have time to see changes.
+      Useful companion of headless option, so that you have time to see changes.
   * `:timeout` (Numeric) - The number of seconds we'll wait for a response when
       communicating with browser. Default is 5.
   * `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby.
@@ -601,41 +601,36 @@ browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https:
 
 ## Proxy
 
-You can set a proxy with the `proxy` option.
+You can set a proxy with a `:proxy` option:
 
 ```ruby
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800" })
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", pasword: "pa$$" })
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", password: "pa$$" })
 ```
 
-Chrome Devtools Protocol does not support changing proxies after the browser is launched. If you want to change proxies, you must restart your browser, which may not be convenient. There is a workaround. Ferrum provides a wrapper for a proxy server that can rotate proxies. We can run a proxy in the same process and rotate proxies inside this proxy server:
+`:bypass` can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
 
 ```ruby
-browser = Ferrum::Browser.new(proxy: { server: true })
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
+```
+
+In general passing a proxy option when instantiating a browser results in a browser running with proxy command line
+flags, so that it affects all pages and contexts. You can create a page in a new context which can use its own proxy
+settings:
+
+```ruby
+browser = Ferrum::Browser.new
 
-browser.proxy_server.rotate(host: "x.x.x.x", port: 31337, user: "user", password: "password")
-browser.create_page(new_context: true) do |page|
+browser.create_page(proxy: { host: "x.x.x.x", port: 31337, user: "user", password: "password" }) do |page|
   page.go_to("https://api.ipify.org?format=json")
   page.body # => "x.x.x.x"
 end
 
-browser.proxy_server.rotate(host: "y.y.y.y", port: 31337, user: "user", password: "password")
-browser.create_page(new_context: true) do |page|
+browser.create_page(proxy: { host: "y.y.y.y", port: 31337, user: "user", password: "password" }) do |page|
   page.go_to("https://api.ipify.org?format=json")
   page.body # => "y.y.y.y"
 end
 ```
 
-Make sure to create page in the new context, because Chrome doesn't break the connection with the proxy for `CONNECT`
-requests even if you close the page.
-
-You can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
-
-```ruby
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
-browser = Ferrum::Browser.new(proxy: { server: true, bypass: "*.google.com;*foo.com" })
-```
-
 
 ### Mouse
 

data/lib/ferrum/browser/client.rb

@@ -8,11 +8,11 @@ module Ferrum
     class Client
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
-      def initialize(browser, ws_url, id_starts_with: 0)
-        @browser = browser
+      def initialize(ws_url, connectable, logger: nil, ws_max_receive_size: nil, id_starts_with: 0)
+        @connectable = connectable
         @command_id = id_starts_with
         @pendings = Concurrent::Hash.new
-        @ws = WebSocket.new(ws_url, @browser.ws_max_receive_size, @browser.logger)
+        @ws = WebSocket.new(ws_url, ws_max_receive_size, logger)
         @subscriber, @interrupter = Subscriber.build(2)
 
         @thread = Thread.new do
@@ -39,7 +39,7 @@ module Ferrum
         message = build_message(method, params)
         @pendings[message[:id]] = pending
         @ws.send_message(message)
-        data = pending.value!(@browser.timeout)
+        data = pending.value!(@connectable.timeout)
         @pendings.delete(message[:id])
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?

data/lib/ferrum/browser/command.rb

@@ -10,7 +10,7 @@ module Ferrum
       # Currently only these browsers support CDP:
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
       def self.build(options, user_data_dir)
-        defaults = case options[:browser_name]
+        defaults = case options.browser_name
                    when :firefox
                      Options::Firefox.options
                    when :chrome, :opera, :edge, nil
@@ -29,14 +29,14 @@ module Ferrum
         @defaults = defaults
         @options = options
         @user_data_dir = user_data_dir
-        @path = options[:browser_path] || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
+        @path = options.browser_path || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
         raise BinaryNotFoundError, NOT_FOUND unless @path
 
         merge_options
       end
 
       def xvfb?
-        !!options[:xvfb]
+        !!options.xvfb
       end
 
       def to_a
@@ -47,9 +47,8 @@ module Ferrum
 
       def merge_options
         @flags = defaults.merge_required(@flags, options, @user_data_dir)
-        @flags = defaults.merge_default(@flags, options) unless options[:ignore_default_browser_options]
-
-        @flags.merge!(options.fetch(:browser_options, {}))
+        @flags = defaults.merge_default(@flags, options) unless options.ignore_default_browser_options
+        @flags.merge!(options.browser_options)
       end
     end
   end

data/lib/ferrum/browser/options/base.rb

@@ -4,11 +4,8 @@ require "singleton"
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Base
-        BROWSER_HOST = "127.0.0.1"
-        BROWSER_PORT = "0"
-
         include Singleton
 
         def self.options

data/lib/ferrum/browser/options/chrome.rb

@@ -2,7 +2,7 @@
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Chrome < Base
         DEFAULT_OPTIONS = {
           "headless" => nil,
@@ -59,17 +59,22 @@ module Ferrum
         }.freeze
 
         def merge_required(flags, options, user_data_dir)
-          port = options.fetch(:port, BROWSER_PORT)
-          host = options.fetch(:host, BROWSER_HOST)
-          flags.merge("remote-debugging-port" => port,
-                      "remote-debugging-address" => host,
-                      # Doesn't work on MacOS, so we need to set it by CDP
-                      "window-size" => options[:window_size]&.join(","),
-                      "user-data-dir" => user_data_dir)
+          flags = flags.merge("remote-debugging-port" => options.port,
+                              "remote-debugging-address" => options.host,
+                              # Doesn't work on MacOS, so we need to set it by CDP
+                              "window-size" => options.window_size&.join(","),
+                              "user-data-dir" => user_data_dir)
+
+          if options.proxy
+            flags.merge!("proxy-server" => "#{options.proxy[:host]}:#{options.proxy[:port]}")
+            flags.merge!("proxy-bypass-list" => options.proxy[:bypass]) if options.proxy[:bypass]
+          end
+
+          flags
         end
 
         def merge_default(flags, options)
-          defaults = except("headless", "disable-gpu") unless options.fetch(:headless, true)
+          defaults = except("headless", "disable-gpu") unless options.headless
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/options/firefox.rb

@@ -2,7 +2,7 @@
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Firefox < Base
         DEFAULT_OPTIONS = {
           "headless" => nil
@@ -23,14 +23,11 @@ module Ferrum
         }.freeze
 
         def merge_required(flags, options, user_data_dir)
-          port = options.fetch(:port, BROWSER_PORT)
-          host = options.fetch(:host, BROWSER_HOST)
-          flags.merge("remote-debugger" => "#{host}:#{port}",
-                      "profile" => user_data_dir)
+          flags.merge("remote-debugger" => "#{options.host}:#{options.port}", "profile" => user_data_dir)
         end
 
         def merge_default(flags, options)
-          defaults = except("headless") unless options.fetch(:headless, true)
+          defaults = except("headless") unless options.headless
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/options.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Options
+      HEADLESS = true
+      BROWSER_PORT = "0"
+      BROWSER_HOST = "127.0.0.1"
+      WINDOW_SIZE = [1024, 768].freeze
+      BASE_URL_SCHEMA = %w[http https].freeze
+      DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
+      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
+      DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
+
+      attr_reader :window_size, :timeout, :logger, :ws_max_receive_size,
+                  :js_errors, :base_url, :slowmo, :pending_connection_errors,
+                  :url, :env, :process_timeout, :browser_name, :browser_path,
+                  :save_path, :extensions, :proxy, :port, :host, :headless,
+                  :ignore_default_browser_options, :browser_options, :xvfb
+
+      def initialize(options = nil)
+        @options = Hash(options&.dup)
+        @port = @options.fetch(:port, BROWSER_PORT)
+        @host = @options.fetch(:host, BROWSER_HOST)
+        @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
+        @window_size = @options.fetch(:window_size, WINDOW_SIZE)
+        @js_errors = @options.fetch(:js_errors, false)
+        @headless = @options.fetch(:headless, HEADLESS)
+        @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+        @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
+        @browser_options = @options.fetch(:browser_options, {})
+        @slowmo = @options[:slowmo].to_f
+
+        @ws_max_receive_size, @env, @browser_name, @browser_path,
+          @save_path, @extensions, @ignore_default_browser_options, @xvfb = @options.values_at(
+            :ws_max_receive_size, :env, :browser_name, :browser_path, :save_path, :extensions,
+            :ignore_default_browser_options, :xvfb
+          )
+
+        @options[:window_size] = @window_size
+        @proxy = parse_proxy(@options[:proxy])
+        @logger = parse_logger(@options[:logger])
+        @base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
+        @url = @options[:url].to_s if @options[:url]
+
+        @options.freeze
+        @browser_options.freeze
+      end
+
+      def to_h
+        @options
+      end
+
+      def parse_base_url(value)
+        parsed = Addressable::URI.parse(value)
+        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
+          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+        end
+
+        parsed
+      end
+
+      def parse_proxy(options)
+        return unless options
+
+        raise ArgumentError, "proxy options must be a Hash" unless options.is_a?(Hash)
+
+        if options[:host].nil? && options[:port].nil?
+          raise ArgumentError, "proxy options must be a Hash with at least :host | :port"
+        end
+
+        options
+      end
+
+      private
+
+      def parse_logger(logger)
+        return logger if logger
+
+        !logger && DEBUG_MODE ? $stdout.tap { |s| s.sync = true } : logger
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -15,7 +15,6 @@ module Ferrum
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
-      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
 
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
@@ -63,17 +62,17 @@ module Ferrum
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
 
-        if options[:url]
-          url = URI.join(options[:url].to_s, "/json/version")
+        if options.url
+          url = URI.join(options.url, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
           self.ws_url = response["webSocketDebuggerUrl"]
           parse_browser_versions
           return
         end
 
-        @logger = options[:logger]
-        @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
-        @env = Hash(options[:env])
+        @logger = options.logger
+        @process_timeout = options.process_timeout
+        @env = Hash(options.env)
 
         tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))

data/lib/ferrum/browser/version_info.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    #
+    # The browser's version information returned by [Browser.getVersion].
+    #
+    # [Browser.getVersion]: https://chromedevtools.github.io/devtools-protocol/1-3/Browser/#method-getVersion
+    #
+    # @since 0.13
+    #
+    class VersionInfo
+      #
+      # Initializes the browser's version information.
+      #
+      # @param [Hash{String => Object}] properties
+      #   The object properties returned by [Browser.getVersion](https://chromedevtools.github.io/devtools-protocol/1-3/Browser/#method-getVersion).
+      #
+      # @api private
+      #
+      def initialize(properties)
+        @properties = properties
+      end
+
+      #
+      # The Chrome DevTools protocol version.
+      #
+      # @return [String]
+      #
+      def protocol_version
+        @properties["protocolVersion"]
+      end
+
+      #
+      # The Chrome version.
+      #
+      # @return [String]
+      #
+      def product
+        @properties["product"]
+      end
+
+      #
+      # The Chrome revision properties.
+      #
+      # @return [String]
+      #
+      def revision
+        @properties["revision"]
+      end
+
+      #
+      # The Chrome `User-Agent` string.
+      #
+      # @return [String]
+      #
+      def user_agent
+        @properties["userAgent"]
+      end
+
+      #
+      # The JavaScript engine version.
+      #
+      # @return [String]
+      #
+      def js_version
+        @properties["jsVersion"]
+      end
+    end
+  end
+end

data/lib/ferrum/browser/xvfb.rb

@@ -16,7 +16,7 @@ module Ferrum
         @path = Binary.find("Xvfb")
         raise BinaryNotFoundError, NOT_FOUND unless @path
 
-        @screen_size = "#{options.fetch(:window_size, [1024, 768]).join('x')}x24"
+        @screen_size = "#{options.window_size.join('x')}x24"
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end