ferrum 0.12 → 0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e4ca57662283c8c6b917aa3e2f1f06b154cb66ffdd8522756f55e09be84a423
-  data.tar.gz: 50d37d615399fe481634154d619d55872ee13b0cace511a1b147da38454247b5
+  metadata.gz: 6cbef2f4caac663a8b39afc37b94e46f00c9471cdaea36a1781dcfa99488a546
+  data.tar.gz: 58d8f68e84138b71de15ac33368d4cd4f8992db4ab7f321a3e3f03530968502b
 SHA512:
-  metadata.gz: 9cc514740ae8041b61a184bdeee0a44ce26b276f36af0358c6a00524f8e13b6db9ddda48acf5781c4122acaa4980579cf06053ffb420c2013e0b44177eab2571
-  data.tar.gz: cc00b6d3cb440db600241156354dcd92a6bf67c0402b9e2881f32ea507549d80f69b51011bb246baf9faf95f33e5596933ce569cd9b03f41c7cc066d215b6073
+  metadata.gz: a621c8e99fe1ec5e183ad54681da4a6bf09105fc66477c37b76b59cb1842f679ac679690514de1829fbbe18af4b81e0e01a64b564a48db9c49af151fc5b64e51
+  data.tar.gz: cc3089d1b9a81d43e8dbeb356b0d677d5377bc1d1d4667e0da9f389034e044ef0497ced614b39812cff53dd70930ec1f66e473f2a30a7edad825dd3424df7e54

data/README.md

@@ -134,6 +134,7 @@ In docker as root you must pass the no-sandbox browser option:
 Ferrum::Browser.new(browser_options: { 'no-sandbox': nil })
 ```
 
+It has also been reported that the Chrome process repeatedly crashes when running inside a Docker container on an M1 Mac preventing Ferrum from working. Ferrum should work as expected when deployed to a Docker container on a non-M1 Mac.
 
 ## Customization
 
@@ -144,7 +145,8 @@ Ferrum::Browser.new(options)
 ```
 
 * options `Hash`
-  * `:headless` (Boolean) - Set browser as headless or not, `true` by default.
+  * `:headless` (String | Boolean) - Set browser as headless or not, `true` by default. You can set `"new"` to support
+      [new headless mode](https://developer.chrome.com/articles/new-headless/).
   * `:xvfb` (Boolean) - Run browser in a virtual framebuffer, `false` by default.
   * `:window_size` (Array) - The dimensions of the browser window in which to
       test, expressed as a 2-element array, e.g. [1024, 768]. Default: [1024, 768]
@@ -154,7 +156,7 @@ Ferrum::Browser.new(options)
   * `:logger` (Object responding to `puts`) - When present, debug output is
       written to this object.
   * `:slowmo` (Integer | Float) - Set a delay in seconds to wait before sending command.
-      Usefull companion of headless option, so that you have time to see changes.
+      Useful companion of headless option, so that you have time to see changes.
   * `:timeout` (Numeric) - The number of seconds we'll wait for a response when
       communicating with browser. Default is 5.
   * `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby.
@@ -595,47 +597,49 @@ Activates offline mode for a page.
 
 ```ruby
 browser.network.offline_mode
-browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed to reach server, check DNS and server status)
+browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed(net::ERR_INTERNET_DISCONNECTED))
 ```
 
+#### cache(disable: `Boolean`)
+
+Toggles ignoring cache for each request. If true, cache will not be used.
+
+```ruby
+browser.network.cache(disable: true)
+```
 
 ## Proxy
 
-You can set a proxy with the `proxy` option.
+You can set a proxy with a `:proxy` option:
+
+```ruby
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", password: "pa$$" })
+```
+
+`:bypass` can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
 
 ```ruby
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800" })
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", pasword: "pa$$" })
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
 ```
 
-Chrome Devtools Protocol does not support changing proxies after the browser is launched. If you want to change proxies, you must restart your browser, which may not be convenient. There is a workaround. Ferrum provides a wrapper for a proxy server that can rotate proxies. We can run a proxy in the same process and rotate proxies inside this proxy server:
+In general passing a proxy option when instantiating a browser results in a browser running with proxy command line
+flags, so that it affects all pages and contexts. You can create a page in a new context which can use its own proxy
+settings:
 
 ```ruby
-browser = Ferrum::Browser.new(proxy: { server: true })
+browser = Ferrum::Browser.new
 
-browser.proxy_server.rotate(host: "x.x.x.x", port: 31337, user: "user", password: "password")
-browser.create_page(new_context: true) do |page|
+browser.create_page(proxy: { host: "x.x.x.x", port: 31337, user: "user", password: "password" }) do |page|
   page.go_to("https://api.ipify.org?format=json")
   page.body # => "x.x.x.x"
 end
 
-browser.proxy_server.rotate(host: "y.y.y.y", port: 31337, user: "user", password: "password")
-browser.create_page(new_context: true) do |page|
+browser.create_page(proxy: { host: "y.y.y.y", port: 31337, user: "user", password: "password" }) do |page|
   page.go_to("https://api.ipify.org?format=json")
   page.body # => "y.y.y.y"
 end
 ```
 
-Make sure to create page in the new context, because Chrome doesn't break the connection with the proxy for `CONNECT`
-requests even if you close the page.
-
-You can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
-
-```ruby
-browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
-browser = Ferrum::Browser.new(proxy: { server: true, bypass: "*.google.com;*foo.com" })
-```
-
 
 ### Mouse
 
@@ -1133,6 +1137,8 @@ frame.at_css("//a[text() = 'Log in']") # => Node
 #### evaluate
 #### selected : `Array<Node>`
 #### select
+#### scroll_into_view
+#### in_viewport?(of: `Node | nil`) : `Boolean`
 
 (chainable) Selects options by passed attribute.
 

data/lib/ferrum/browser/client.rb

@@ -8,11 +8,11 @@ module Ferrum
     class Client
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
-      def initialize(browser, ws_url, id_starts_with: 0)
-        @browser = browser
+      def initialize(ws_url, connectable, logger: nil, ws_max_receive_size: nil, id_starts_with: 0)
+        @connectable = connectable
         @command_id = id_starts_with
         @pendings = Concurrent::Hash.new
-        @ws = WebSocket.new(ws_url, @browser.ws_max_receive_size, @browser.logger)
+        @ws = WebSocket.new(ws_url, ws_max_receive_size, logger)
         @subscriber, @interrupter = Subscriber.build(2)
 
         @thread = Thread.new do
@@ -39,7 +39,7 @@ module Ferrum
         message = build_message(method, params)
         @pendings[message[:id]] = pending
         @ws.send_message(message)
-        data = pending.value!(@browser.timeout)
+        data = pending.value!(@connectable.timeout)
         @pendings.delete(message[:id])
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
@@ -84,7 +84,8 @@ module Ferrum
         case error["message"]
         # Node has disappeared while we were trying to get it
         when "No node with given id found",
-             "Could not find node with given id"
+             "Could not find node with given id",
+             "Inspected target navigated or closed"
           raise NodeNotFoundError, error
         # Context is lost, page is reloading
         when "Cannot find context with specified id"

data/lib/ferrum/browser/command.rb

@@ -10,7 +10,7 @@ module Ferrum
       # Currently only these browsers support CDP:
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
       def self.build(options, user_data_dir)
-        defaults = case options[:browser_name]
+        defaults = case options.browser_name
                    when :firefox
                      Options::Firefox.options
                    when :chrome, :opera, :edge, nil
@@ -29,14 +29,18 @@ module Ferrum
         @defaults = defaults
         @options = options
         @user_data_dir = user_data_dir
-        @path = options[:browser_path] || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
+        @path = options.browser_path || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
         raise BinaryNotFoundError, NOT_FOUND unless @path
 
         merge_options
       end
 
       def xvfb?
-        !!options[:xvfb]
+        !!options.xvfb
+      end
+
+      def headless_new?
+        @flags["headless"] == "new"
       end
 
       def to_a
@@ -47,9 +51,8 @@ module Ferrum
 
       def merge_options
         @flags = defaults.merge_required(@flags, options, @user_data_dir)
-        @flags = defaults.merge_default(@flags, options) unless options[:ignore_default_browser_options]
-
-        @flags.merge!(options.fetch(:browser_options, {}))
+        @flags = defaults.merge_default(@flags, options) unless options.ignore_default_browser_options
+        @flags.merge!(options.browser_options)
       end
     end
   end

data/lib/ferrum/browser/options/base.rb

@@ -4,11 +4,8 @@ require "singleton"
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Base
-        BROWSER_HOST = "127.0.0.1"
-        BROWSER_PORT = "0"
-
         include Singleton
 
         def self.options

data/lib/ferrum/browser/options/chrome.rb

@@ -2,7 +2,7 @@
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Chrome < Base
         DEFAULT_OPTIONS = {
           "headless" => nil,
@@ -19,8 +19,9 @@ module Ferrum
           "keep-alive-for-test" => nil,
           "disable-popup-blocking" => nil,
           "disable-extensions" => nil,
+          "disable-component-extensions-with-background-pages" => nil,
           "disable-hang-monitor" => nil,
-          "disable-features" => "site-per-process,TranslateUI",
+          "disable-features" => "site-per-process,IsolateOrigins,TranslateUI",
           "disable-translate" => nil,
           "disable-background-networking" => nil,
           "enable-features" => "NetworkService,NetworkServiceInProcess",
@@ -32,6 +33,7 @@ module Ferrum
           "disable-ipc-flooding-protection" => nil,
           "disable-prompt-on-repost" => nil,
           "disable-renderer-backgrounding" => nil,
+          "disable-site-isolation-trials" => nil,
           "force-color-profile" => "srgb",
           "metrics-recording-only" => nil,
           "safebrowsing-disable-auto-update" => nil,
@@ -59,17 +61,27 @@ module Ferrum
         }.freeze
 
         def merge_required(flags, options, user_data_dir)
-          port = options.fetch(:port, BROWSER_PORT)
-          host = options.fetch(:host, BROWSER_HOST)
-          flags.merge("remote-debugging-port" => port,
-                      "remote-debugging-address" => host,
-                      # Doesn't work on MacOS, so we need to set it by CDP
-                      "window-size" => options[:window_size]&.join(","),
-                      "user-data-dir" => user_data_dir)
+          flags = flags.merge("remote-debugging-port" => options.port,
+                              "remote-debugging-address" => options.host,
+                              # Doesn't work on MacOS, so we need to set it by CDP
+                              "window-size" => options.window_size&.join(","),
+                              "user-data-dir" => user_data_dir)
+
+          if options.proxy
+            flags.merge!("proxy-server" => "#{options.proxy[:host]}:#{options.proxy[:port]}")
+            flags.merge!("proxy-bypass-list" => options.proxy[:bypass]) if options.proxy[:bypass]
+          end
+
+          flags
         end
 
         def merge_default(flags, options)
-          defaults = except("headless", "disable-gpu") unless options.fetch(:headless, true)
+          defaults = case options.headless
+                     when false
+                       except("headless", "disable-gpu")
+                     when "new"
+                       except("headless").merge("headless" => "new")
+                     end
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/options/firefox.rb

@@ -2,7 +2,7 @@
 
 module Ferrum
   class Browser
-    module Options
+    class Options
       class Firefox < Base
         DEFAULT_OPTIONS = {
           "headless" => nil
@@ -23,14 +23,11 @@ module Ferrum
         }.freeze
 
         def merge_required(flags, options, user_data_dir)
-          port = options.fetch(:port, BROWSER_PORT)
-          host = options.fetch(:host, BROWSER_HOST)
-          flags.merge("remote-debugger" => "#{host}:#{port}",
-                      "profile" => user_data_dir)
+          flags.merge("remote-debugger" => "#{options.host}:#{options.port}", "profile" => user_data_dir)
         end
 
         def merge_default(flags, options)
-          defaults = except("headless") unless options.fetch(:headless, true)
+          defaults = except("headless") unless options.headless
 
           defaults ||= DEFAULT_OPTIONS
           defaults.merge(flags)

data/lib/ferrum/browser/options.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Options
+      HEADLESS = true
+      BROWSER_PORT = "0"
+      BROWSER_HOST = "127.0.0.1"
+      WINDOW_SIZE = [1024, 768].freeze
+      BASE_URL_SCHEMA = %w[http https].freeze
+      DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
+      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
+      DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
+
+      attr_reader :window_size, :timeout, :logger, :ws_max_receive_size,
+                  :js_errors, :base_url, :slowmo, :pending_connection_errors,
+                  :url, :env, :process_timeout, :browser_name, :browser_path,
+                  :save_path, :extensions, :proxy, :port, :host, :headless,
+                  :ignore_default_browser_options, :browser_options, :xvfb
+
+      def initialize(options = nil)
+        @options = Hash(options&.dup)
+        @port = @options.fetch(:port, BROWSER_PORT)
+        @host = @options.fetch(:host, BROWSER_HOST)
+        @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
+        @window_size = @options.fetch(:window_size, WINDOW_SIZE)
+        @js_errors = @options.fetch(:js_errors, false)
+        @headless = @options.fetch(:headless, HEADLESS)
+        @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+        @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
+        @browser_options = @options.fetch(:browser_options, {})
+        @slowmo = @options[:slowmo].to_f
+
+        @ws_max_receive_size, @env, @browser_name, @browser_path,
+          @save_path, @extensions, @ignore_default_browser_options, @xvfb = @options.values_at(
+            :ws_max_receive_size, :env, :browser_name, :browser_path, :save_path, :extensions,
+            :ignore_default_browser_options, :xvfb
+          )
+
+        @options[:window_size] = @window_size
+        @proxy = parse_proxy(@options[:proxy])
+        @logger = parse_logger(@options[:logger])
+        @base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
+        @url = @options[:url].to_s if @options[:url]
+
+        @options.freeze
+        @browser_options.freeze
+      end
+
+      def to_h
+        @options
+      end
+
+      def parse_base_url(value)
+        parsed = Addressable::URI.parse(value)
+        unless BASE_URL_SCHEMA.include?(parsed&.normalized_scheme)
+          raise ArgumentError, "`base_url` should be absolute and include schema: #{BASE_URL_SCHEMA.join(' | ')}"
+        end
+
+        parsed
+      end
+
+      def parse_proxy(options)
+        return unless options
+
+        raise ArgumentError, "proxy options must be a Hash" unless options.is_a?(Hash)
+
+        if options[:host].nil? && options[:port].nil?
+          raise ArgumentError, "proxy options must be a Hash with at least :host | :port"
+        end
+
+        options
+      end
+
+      private
+
+      def parse_logger(logger)
+        return logger if logger
+
+        !logger && DEBUG_MODE ? $stdout.tap { |s| s.sync = true } : logger
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -15,7 +15,6 @@ module Ferrum
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
-      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
 
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
@@ -63,17 +62,17 @@ module Ferrum
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
 
-        if options[:url]
-          url = URI.join(options[:url].to_s, "/json/version")
+        if options.url
+          url = URI.join(options.url, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
           self.ws_url = response["webSocketDebuggerUrl"]
           parse_browser_versions
           return
         end
 
-        @logger = options[:logger]
-        @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
-        @env = Hash(options[:env])
+        @logger = options.logger
+        @process_timeout = options.process_timeout
+        @env = Hash(options.env)
 
         tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
@@ -138,7 +137,7 @@ module Ferrum
         output = ""
         start = Utils::ElapsedTime.monotonic_time
         max_time = start + timeout
-        regexp = %r{DevTools listening on (ws://.*)}
+        regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
         while (now = Utils::ElapsedTime.monotonic_time) < max_time
           begin
             output += read_io.read_nonblock(512)

data/lib/ferrum/browser/version_info.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    #
+    # The browser's version information returned by [Browser.getVersion].
+    #
+    # [Browser.getVersion]: https://chromedevtools.github.io/devtools-protocol/1-3/Browser/#method-getVersion
+    #
+    # @since 0.13
+    #
+    class VersionInfo
+      #
+      # Initializes the browser's version information.
+      #
+      # @param [Hash{String => Object}] properties
+      #   The object properties returned by [Browser.getVersion](https://chromedevtools.github.io/devtools-protocol/1-3/Browser/#method-getVersion).
+      #
+      # @api private
+      #
+      def initialize(properties)
+        @properties = properties
+      end
+
+      #
+      # The Chrome DevTools protocol version.
+      #
+      # @return [String]
+      #
+      def protocol_version
+        @properties["protocolVersion"]
+      end
+
+      #
+      # The Chrome version.
+      #
+      # @return [String]
+      #
+      def product
+        @properties["product"]
+      end
+
+      #
+      # The Chrome revision properties.
+      #
+      # @return [String]
+      #
+      def revision
+        @properties["revision"]
+      end
+
+      #
+      # The Chrome `User-Agent` string.
+      #
+      # @return [String]
+      #
+      def user_agent
+        @properties["userAgent"]
+      end
+
+      #
+      # The JavaScript engine version.
+      #
+      # @return [String]
+      #
+      def js_version
+        @properties["jsVersion"]
+      end
+    end
+  end
+end

data/lib/ferrum/browser/web_socket.rb

@@ -7,7 +7,7 @@ require "websocket/driver"
 module Ferrum
   class Browser
     class WebSocket
-      WEBSOCKET_BUG_SLEEP = 0.01
+      WEBSOCKET_BUG_SLEEP = 0.05
       SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
 
       attr_reader :url, :messages

data/lib/ferrum/browser/xvfb.rb

@@ -16,7 +16,7 @@ module Ferrum
         @path = Binary.find("Xvfb")
         raise BinaryNotFoundError, NOT_FOUND unless @path
 
-        @screen_size = "#{options.fetch(:window_size, [1024, 768]).join('x')}x24"
+        @screen_size = "#{options.window_size.join('x')}x24"
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end