ferrum 0.6.2 → 0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c0325708ecf4ff29cd0d11a3527207271a1a9a76c945a321810b7f37325c538
-  data.tar.gz: 9d2f4790ef55ddfb55f1588473dc94511fc95fe7c7cc937f0e49af0980457b9e
+  metadata.gz: 34f864b5679986d8580fee118735ea2bf62b5b35f1d2ae9f403cdc08168d1d48
+  data.tar.gz: 5899a23fdf4219dfd7aa70b7f1d72d0dcee167a6102b07a5062c36b8c4588f90
 SHA512:
-  metadata.gz: 8067b96de131a957317d641afeec6d145f47455b18eb119227dc1de184ed35b8b6a6cbf84aa8e1e6b3c87ff190b2218c68dd43cb1c9187fe7e7acea2cae1ca89
-  data.tar.gz: a4fb8e6749da0fa5ccbf1ab05c807f176e30c2650e23b263072281065b5ca9632201a596652989636f7b9f565ac8a23ca391e8d366d755538e7b28504b09209b
+  metadata.gz: 3732e2ba120edbd7e1d28759d6d9fe75c4416f41cc08a535e63751324faec35117a805de3dfdb1467fa3aa55c2885b8c7058d1fce46bb0b69e642210f89bac26
+  data.tar.gz: 426b53078a92ddc04187250335c200dd35dac489eb2fe887f07a7ae8bb07f0652588d278ef18d7389221f692ae3b55488cedfcec0ea8511b57d9ebc9776060db

data/README.md

@@ -12,12 +12,31 @@ It is Ruby clean and high-level API to Chrome. Runs headless by default,
 but you can configure it to run in a non-headless mode. All you need is Ruby and
 Chrome/Chromium. Ferrum connects to the browser via DevTools Protocol.
 
-Relation to [Cuprite](https://github.com/machinio/cuprite). Cuprite used to have
-this code inside in one form or another but the thing is you don't need capybara
-if you are going to crawl sites. You crawl, not test. Besides that clean
-lightweight API to browser is what Ruby was missing, so here it comes.
-
-If you like this project, please consider to [become a backer](https://www.patreon.com/rferrum) on Patreon.
+[Cuprite](https://github.com/machinio/cuprite) used to have this code inside in
+one form or another but the thing is you don't need Capybara if you are going to
+crawl sites. You crawl, not test. Besides that clean lightweight API to browser
+is what Ruby was missing, so here it comes.
+
+[Vessel](https://github.com/route/vessel) high-level web crawling framework
+based on Ferrum.
+
+If you like this project, please consider to _[become a backer](https://www.patreon.com/rferrum)_
+on Patreon.
+
+## Index
+
+* [Customization](https://github.com/route/ferrum#customization)
+* [Navigation](https://github.com/route/ferrum#navigation)
+* [Finders](https://github.com/route/ferrum#finders)
+* [Screenshots](https://github.com/route/ferrum#screenshots)
+* [Network](https://github.com/route/ferrum#network)
+* [Mouse](https://github.com/route/ferrum#mouse)
+* [Keyboard](https://github.com/route/ferrum#keyboard)
+* [Cookies](https://github.com/route/ferrum#cookies)
+* [Headers](https://github.com/route/ferrum#headers)
+* [JavaScript](https://github.com/route/ferrum#javascript)
+* [Frames](https://github.com/route/ferrum#frames)
+* [Dialog](https://github.com/route/ferrum#dialog)
 
 ## Install
 
@@ -48,8 +67,8 @@ Interact with a page:
 browser = Ferrum::Browser.new
 browser.goto("https://google.com")
 input = browser.at_xpath("//div[@id='searchform']/form//input[@type='text']")
-input.focus.type("Ruby headless driver for Capybara", :Enter)
-browser.at_css("a > h3").text # => "machinio/cuprite: Headless Chrome driver for Capybara - GitHub"
+input.focus.type("Ruby headless driver for Chrome", :Enter)
+browser.at_css("a > h3").text # => "route/ferrum: Ruby Chrome/Chromium driver - GitHub"
 browser.quit
 ```
 
@@ -93,22 +112,26 @@ Ferrum::Browser.new(options)
 ```
 
 * options `Hash`
-  * `:browser_path` (String) - Path to chrome binary, you can also set ENV
-      variable as `BROWSER_PATH=some/path/chrome bundle exec rspec`.
   * `:headless` (Boolean) - Set browser as headless or not, `true` by default.
-  * `:slowmo` (Integer | Float) - Set a delay to wait before sending command.
-      Usefull companion of headless option, so that you have time to see changes.
+  * `:window_size` (Array) - The dimensions of the browser window in which to
+      test, expressed as a 2-element array, e.g. [1024, 768]. Default: [1024, 768]
+  * `:extensions` (Array[String | Hash]) - An array of paths to files or JS
+      source code to be preloaded into the browser e.g.:
+      `["/path/to/script.js", { source: "window.secret = 'top'" }]`
   * `:logger` (Object responding to `puts`) - When present, debug output is
       written to this object.
+  * `:slowmo` (Integer | Float) - Set a delay to wait before sending command.
+      Usefull companion of headless option, so that you have time to see changes.
   * `:timeout` (Numeric) - The number of seconds we'll wait for a response when
       communicating with browser. Default is 5.
   * `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby.
-  * `:window_size` (Array) - The dimensions of the browser window in which to
-      test, expressed as a 2-element array, e.g. [1024, 768]. Default: [1024, 768]
+  * `:browser_name` (Symbol) - `:chrome` by default, only experimental support
+      for `:firefox` for now.
+  * `:browser_path` (String) - Path to chrome binary, you can also set ENV
+      variable as `BROWSER_PATH=some/path/chrome bundle exec rspec`.
   * `:browser_options` (Hash) - Additional command line options,
       [see them all](https://peter.sh/experiments/chromium-command-line-switches/)
       e.g. `{ "ignore-certificate-errors" => nil }`
-  * `:extensions` (Array) - An array of JS files to be preloaded into the browser
   * `:port` (Integer) - Remote debugging port for headless Chrome
   * `:host` (String) - Remote debugging address for headless Chrome
   * `:url` (String) - URL for a running instance of Chrome. If this is set, a
@@ -342,6 +365,24 @@ browser.goto("https://github.com/")
 browser.network.status # => 200
 ```
 
+#### wait_for_idle(\*\*options)
+
+Waits for network idle or raises `Ferrum::TimeoutError` error
+
+* options `Hash`
+  * :connections `Integer` how many connections are allowed for network to be
+    idling, `0` by default
+  * :duration `Float` sleep for given amount of time and check again, `0.05` by
+    default
+  * :timeout `Float` during what time we try to check idle, `browser.timeout`
+    by default
+
+```ruby
+browser.goto("https://example.com/")
+browser.at_xpath("//a[text() = 'No UI changes button']").click
+browser.network.wait_for_idle
+```
+
 #### clear(type)
 
 Clear browser's cache or collected traffic.
@@ -628,6 +669,18 @@ browser.add_script_tag(url: "http://example.com/stylesheet.css") # => true
 
 ```ruby
 browser.add_style_tag(content: "h1 { font-size: 40px; }") # => true
+
+```
+#### bypass_csp(enabled) : `Boolean`
+
+* enabled `Boolean`, `true` by default
+
+```ruby
+browser.bypass_csp # => true
+browser.goto("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
+browser.refresh
+browser.add_script_tag(content: "window.__injected = 42")
+browser.evaluate("window.__injected") # => 42
 ```
 
 

data/lib/ferrum.rb

@@ -10,8 +10,14 @@ module Ferrum
   class NotImplementedError < Error; end
 
   class StatusError < Error
-    def initialize(url)
-      super("Request to #{url} failed to reach server, check DNS and/or server status")
+    def initialize(url, pendings = [])
+      message = if pendings.empty?
+                  "Request to #{url} failed to reach server, check DNS and/or server status"
+                else
+                  "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+                end
+
+      super(message)
     end
   end
 
@@ -31,7 +37,7 @@ module Ferrum
   end
 
   class DeadBrowserError < Error
-    def initialize(message = "Browser is dead")
+    def initialize(message = "Browser is dead or given window is closed")
       super
     end
   end

data/lib/ferrum/browser.rb

@@ -16,15 +16,16 @@ module Ferrum
     extend Forwardable
     delegate %i[default_context] => :contexts
     delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[goto back forward refresh
+    delegate %i[goto back forward refresh reload
                 at_css at_xpath css xpath current_url title body doctype
                 headers cookies network
                 mouse keyboard
                 screenshot pdf viewport_size
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute
-                add_script_tag add_style_tag
+                add_script_tag add_style_tag bypass_csp
                 on] => :page
+    delegate %i[default_user_agent] => :process
 
     attr_reader :client, :process, :contexts, :logger, :js_errors,
                 :slowmo, :base_url, :options, :window_size
@@ -67,7 +68,9 @@ module Ferrum
     end
 
     def extensions
-      @extensions ||= Array(@options[:extensions]).map { |p| File.read(p) }
+      @extensions ||= Array(@options[:extensions]).map do |ext|
+        (ext.is_a?(Hash) && ext[:source]) || File.read(ext)
+      end
     end
 
     def timeout

data/lib/ferrum/browser/chrome.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Chrome < Command
+      DEFAULT_OPTIONS = {
+        "headless" => nil,
+        "disable-gpu" => nil,
+        "hide-scrollbars" => nil,
+        "mute-audio" => nil,
+        "enable-automation" => nil,
+        "disable-web-security" => nil,
+        "disable-session-crashed-bubble" => nil,
+        "disable-breakpad" => nil,
+        "disable-sync" => nil,
+        "no-first-run" => nil,
+        "use-mock-keychain" => nil,
+        "keep-alive-for-test" => nil,
+        "disable-popup-blocking" => nil,
+        "disable-extensions" => nil,
+        "disable-hang-monitor" => nil,
+        "disable-features" => "site-per-process,TranslateUI",
+        "disable-translate" => nil,
+        "disable-background-networking" => nil,
+        "enable-features" => "NetworkService,NetworkServiceInProcess",
+        "disable-background-timer-throttling" => nil,
+        "disable-backgrounding-occluded-windows" => nil,
+        "disable-client-side-phishing-detection" => nil,
+        "disable-default-apps" => nil,
+        "disable-dev-shm-usage" => nil,
+        "disable-ipc-flooding-protection" => nil,
+        "disable-prompt-on-repost" => nil,
+        "disable-renderer-backgrounding" => nil,
+        "force-color-profile" => "srgb",
+        "metrics-recording-only" => nil,
+        "safebrowsing-disable-auto-update" => nil,
+        "password-store" => "basic",
+        # Note: --no-sandbox is not needed if you properly setup a user in the container.
+        # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
+        # "no-sandbox" => nil,
+      }.freeze
+
+      MAC_BIN_PATH = [
+        "/Applications/Chromium.app/Contents/MacOS/Chromium",
+        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+      ].freeze
+      LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
+                          google-chrome chrome chromium-browser
+                          google-chrome-stable].freeze
+
+      private
+
+      def combine_flags
+        # Doesn't work on MacOS, so we need to set it by CDP as well
+        @flags.merge!("window-size" => options[:window_size].join(","))
+
+        port = options.fetch(:port, BROWSER_PORT)
+        @flags.merge!("remote-debugging-port" => port)
+
+        host = options.fetch(:host, BROWSER_HOST)
+        @flags.merge!("remote-debugging-address" => host)
+
+        @flags.merge!("user-data-dir" => @user_data_dir)
+
+        @flags = DEFAULT_OPTIONS.merge(@flags)
+
+        unless options.fetch(:headless, true)
+          @flags.delete("headless")
+          @flags.delete("disable-gpu")
+        end
+
+        @flags.merge!(options.fetch(:browser_options, {}))
+      end
+    end
+  end
+end

data/lib/ferrum/browser/client.rb

@@ -7,20 +7,26 @@ require "ferrum/browser/web_socket"
 module Ferrum
   class Browser
     class Client
+      INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
+
       def initialize(browser, ws_url, start_id = 0, allow_slowmo = true)
         @command_id = start_id
         @pendings = Concurrent::Hash.new
         @browser = browser
         @slowmo = @browser.slowmo if allow_slowmo && @browser.slowmo > 0
         @ws = WebSocket.new(ws_url, @browser.logger)
-        @subscriber = Subscriber.new
+        @subscriber, @interruptor = Subscriber.build(2)
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
+          if Thread.current.respond_to?(:report_on_exception=)
+            Thread.current.report_on_exception = true
+          end
 
           while message = @ws.messages.pop
-            if message.key?("method")
+            if INTERRUPTIONS.include?(message["method"])
+              @interruptor.async.call(message)
+            elsif message.key?("method")
               @subscriber.async.call(message)
             else
               @pendings[message["id"]]&.set(message)
@@ -46,7 +52,12 @@ module Ferrum
       end
 
       def on(event, &block)
-        @subscriber.on(event, &block)
+        case event
+        when *INTERRUPTIONS
+          @interruptor.on(event, &block)
+        else
+          @subscriber.on(event, &block)
+        end
       end
 
       def close

data/lib/ferrum/browser/command.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Command
+      BROWSER_HOST = "127.0.0.1"
+      BROWSER_PORT = "0"
+      NOT_FOUND = "Could not find an executable for the browser. Try to make " \
+                  "it available on the PATH or set environment varible for " \
+                  "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
+
+      # Currently only these browsers support CDP:
+      # https://github.com/cyrus-and/chrome-remote-interface#implementations
+      def self.build(options, user_data_dir)
+        case options[:browser_name]
+        when :firefox
+          Firefox
+        when :chrome, :opera, :edge, nil
+          Chrome
+        else
+          raise NotImplementedError, "not supported browser"
+        end.new(options, user_data_dir)
+      end
+
+      attr_reader :path, :flags, :options
+
+      def initialize(options, user_data_dir)
+        @flags = {}
+        @options, @user_data_dir = options, user_data_dir
+        @path = options[:browser_path] || ENV["BROWSER_PATH"] || detect_path
+        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+
+        combine_flags
+      end
+
+      def to_a
+        [path] + flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
+      end
+
+      private
+
+      def detect_path
+        if Ferrum.mac?
+          self.class::MAC_BIN_PATH.find { |b| File.exist?(b) }
+        else
+          self.class::LINUX_BIN_PATH
+            .find { |b| p = Cliver.detect(b) and break(p) }
+        end
+      end
+
+      def combine_flags
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/ferrum/browser/firefox.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Firefox < Command
+      DEFAULT_OPTIONS = {
+        "headless" => nil,
+      }.freeze
+
+      MAC_BIN_PATH = [
+        "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
+      ].freeze
+      LINUX_BIN_PATH = %w[firefox].freeze
+
+      private
+
+      def combine_flags
+        port = options.fetch(:port, BROWSER_PORT)
+        host = options.fetch(:host, BROWSER_HOST)
+        @flags.merge!("remote-debugger" => "#{host}:#{port}")
+
+        @flags.merge!("profile" => @user_data_dir)
+
+        @flags = DEFAULT_OPTIONS.merge(@flags)
+
+        unless options.fetch(:headless, true)
+          @flags.delete("headless")
+        end
+
+        @flags.merge!(options.fetch(:browser_options, {}))
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -5,6 +5,10 @@ require "net/http"
 require "json"
 require "addressable"
 require "tmpdir"
+require "forwardable"
+require "ferrum/browser/command"
+require "ferrum/browser/chrome"
+require "ferrum/browser/firefox"
 
 module Ferrum
   class Browser
@@ -12,52 +16,14 @@ module Ferrum
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
       PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 2).to_i
-      BROWSER_PATH = ENV["BROWSER_PATH"]
-      BROWSER_HOST = "127.0.0.1"
-      BROWSER_PORT = "0"
-      DEFAULT_OPTIONS = {
-        "headless" => nil,
-        "disable-gpu" => nil,
-        "hide-scrollbars" => nil,
-        "mute-audio" => nil,
-        "enable-automation" => nil,
-        "disable-web-security" => nil,
-        "disable-session-crashed-bubble" => nil,
-        "disable-breakpad" => nil,
-        "disable-sync" => nil,
-        "no-first-run" => nil,
-        "use-mock-keychain" => nil,
-        "keep-alive-for-test" => nil,
-        "disable-popup-blocking" => nil,
-        "disable-extensions" => nil,
-        "disable-hang-monitor" => nil,
-        "disable-features" => "site-per-process,TranslateUI",
-        "disable-translate" => nil,
-        "disable-background-networking" => nil,
-        "enable-features" => "NetworkService,NetworkServiceInProcess",
-        "disable-background-timer-throttling" => nil,
-        "disable-backgrounding-occluded-windows" => nil,
-        "disable-client-side-phishing-detection" => nil,
-        "disable-default-apps" => nil,
-        "disable-dev-shm-usage" => nil,
-        "disable-ipc-flooding-protection" => nil,
-        "disable-prompt-on-repost" => nil,
-        "disable-renderer-backgrounding" => nil,
-        "force-color-profile" => "srgb",
-        "metrics-recording-only" => nil,
-        "safebrowsing-disable-auto-update" => nil,
-        "password-store" => "basic",
-        # Note: --no-sandbox is not needed if you properly setup a user in the container.
-        # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
-        # "no-sandbox" => nil,
-      }.freeze
-
-      NOT_FOUND = "Could not find an executable for chrome. Try to make it " \
-                  "available on the PATH or set environment varible for " \
-                  "example BROWSER_PATH=\"/Applications/Chromium.app/Contents/MacOS/Chromium\""
-
-
-      attr_reader :host, :port, :ws_url, :pid, :path, :options, :cmd
+
+      attr_reader :host, :port, :ws_url, :pid, :command,
+                  :default_user_agent, :browser_version, :protocol_version,
+                  :v8_version, :webkit_version
+
+
+      extend Forwardable
+      delegate path: :command
 
       def self.start(*args)
         new(*args).tap(&:start)
@@ -85,65 +51,24 @@ module Ferrum
       end
 
       def self.directory_remover(path)
-        proc do
-          begin
-            FileUtils.remove_entry(path)
-          rescue Errno::ENOENT
-          end
-        end
-      end
-
-      def self.detect_browser_path
-        if RUBY_PLATFORM.include?("darwin")
-          [
-            "/Applications/Chromium.app/Contents/MacOS/Chromium",
-            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
-          ].find { |path| File.exist?(path) }
-        else
-          %w[chromium google-chrome-unstable google-chrome-beta google-chrome chrome chromium-browser google-chrome-stable].reduce(nil) do |path, exe|
-            path = Cliver.detect(exe)
-            break path if path
-          end
-        end
+        proc { FileUtils.remove_entry(path) rescue Errno::ENOENT }
       end
 
       def initialize(options)
-        @options = {}
-
-        @path = options[:browser_path] || BROWSER_PATH || self.class.detect_browser_path
-
         if options[:url]
           url = URI.join(options[:url].to_s, "/json/version")
           response = JSON.parse(::Net::HTTP.get(url))
           set_ws_url(response["webSocketDebuggerUrl"])
+          parse_browser_versions
           return
         end
 
-        # Doesn't work on MacOS, so we need to set it by CDP as well
-        @options.merge!("window-size" => options[:window_size].join(","))
-
-        port = options.fetch(:port, BROWSER_PORT)
-        @options.merge!("remote-debugging-port" => port)
-
-        host = options.fetch(:host, BROWSER_HOST)
-        @options.merge!("remote-debugging-address" => host)
-
-        @temp_user_data_dir = Dir.mktmpdir
-        ObjectSpace.define_finalizer(self, self.class.directory_remover(@temp_user_data_dir))
-        @options.merge!("user-data-dir" => @temp_user_data_dir)
-
-        @options = DEFAULT_OPTIONS.merge(@options)
-
-        unless options.fetch(:headless, true)
-          @options.delete("headless")
-          @options.delete("disable-gpu")
-        end
-
+        @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
 
-        @options.merge!(options.fetch(:browser_options, {}))
-
-        @logger = options[:logger]
+        tmpdir = Dir.mktmpdir
+        ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
+        @command = Command.build(options, tmpdir)
       end
 
       def start
@@ -156,13 +81,11 @@ module Ferrum
           process_options[:pgroup] = true unless Ferrum.windows?
           process_options[:out] = process_options[:err] = write_io
 
-          raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
-
-          @cmd = [@path] + @options.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
-          @pid = ::Process.spawn(*@cmd, process_options)
+          @pid = ::Process.spawn(*@command.to_a, process_options)
           ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
 
           parse_ws_url(read_io, @process_timeout)
+          parse_browser_versions
         ensure
           close_io(read_io, write_io)
         end
@@ -170,7 +93,7 @@ module Ferrum
 
       def stop
         kill if @pid
-        remove_temp_user_data_dir if @temp_user_data_dir
+        remove_user_data_dir if @user_data_dir
         ObjectSpace.undefine_finalizer(self)
       end
 
@@ -186,9 +109,9 @@ module Ferrum
         @pid = nil
       end
 
-      def remove_temp_user_data_dir
-        self.class.directory_remover(@temp_user_data_dir).call
-        @temp_user_data_dir = nil
+      def remove_user_data_dir
+        self.class.directory_remover(@user_data_dir).call
+        @user_data_dir = nil
       end
 
       def parse_ws_url(read_io, timeout)
@@ -211,7 +134,7 @@ module Ferrum
 
         unless ws_url
           @logger.puts output if @logger
-          raise "Chrome process did not produce websocket url within #{timeout} seconds"
+          raise "Browser process did not produce websocket url within #{timeout} seconds"
         end
       end
 
@@ -221,12 +144,25 @@ module Ferrum
         @port = @ws_url.port
       end
 
+      def parse_browser_versions
+        return unless ws_url.is_a?(Addressable::URI)
+
+        version_url = URI.parse(ws_url.merge(scheme: "http", path: "/json/version"))
+        response = JSON.parse(::Net::HTTP.get(version_url))
+
+        @v8_version = response["V8-Version"]
+        @browser_version = response["Browser"]
+        @webkit_version = response["WebKit-Version"]
+        @default_user_agent = response["User-Agent"]
+        @protocol_version = response["Protocol-Version"]
+      end
+
       def close_io(*ios)
         ios.each do |io|
           begin
             io.close unless io.closed?
           rescue IOError
-            raise unless RUBY_ENGINE == 'jruby'
+            raise unless RUBY_ENGINE == "jruby"
           end
         end
       end

data/lib/ferrum/browser/subscriber.rb

@@ -7,6 +7,10 @@ module Ferrum
     class Subscriber
       include Concurrent::Async
 
+      def self.build(size)
+        (0..size).map { new }
+      end
+
       def initialize
         super
         @on = Hash.new { |h, k| h[k] = [] }

data/lib/ferrum/frame.rb

@@ -18,8 +18,6 @@ module Ferrum
     # Can be one of:
     # * started_loading
     # * navigated
-    # * scheduled_navigation
-    # * cleared_scheduled_navigation
     # * stopped_loading
     def state=(value)
       @state = value

data/lib/ferrum/frame/runtime.rb

@@ -232,7 +232,7 @@ module Ferrum
 
                     const seen = [];
                     function detectCycle(obj) {
-                      if (typeof obj === 'object') {
+                      if (typeof obj === "object") {
                         if (seen.indexOf(obj) !== -1) {
                           return true;
                         }

data/lib/ferrum/headers.rb

@@ -40,7 +40,7 @@ module Ferrum
 
     def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
       options = Hash.new
-      options[:userAgent] = user_agent if user_agent
+      options[:userAgent] = user_agent || @page.browser.default_user_agent
       options[:acceptLanguage] = accept_language if accept_language
       options[:platform] if platform
 

data/lib/ferrum/network.rb

@@ -3,6 +3,9 @@
 require "ferrum/network/exchange"
 require "ferrum/network/intercepted_request"
 require "ferrum/network/auth_request"
+require "ferrum/network/error"
+require "ferrum/network/request"
+require "ferrum/network/response"
 
 module Ferrum
   class Network
@@ -20,6 +23,31 @@ module Ferrum
       @exchange = nil
     end
 
+    def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
+      start = Ferrum.monotonic_time
+
+      until idle?(connections)
+        raise TimeoutError if Ferrum.timeout?(start, timeout)
+        sleep(duration)
+      end
+    end
+
+    def idle?(connections = 0)
+      pending_connections <= connections
+    end
+
+    def total_connections
+      @traffic.size
+    end
+
+    def finished_connections
+      @traffic.count(&:finished?)
+    end
+
+    def pending_connections
+      total_connections - finished_connections
+    end
+
     def request
       @exchange&.request
     end
@@ -87,27 +115,43 @@ module Ferrum
 
     def subscribe
       @page.on("Network.requestWillBeSent") do |params|
+        request = Network::Request.new(params)
+
+        # We can build exchange in two places, here on the event or when request
+        # is interrupted. So we have to be careful when to create new one. We
+        # create new exchange only if there's no with such id or there's but
+        # it's filled with request which means this one is new but has response
+        # for a redirect. So we assign response from the params to previous
+        # exchange and build new exchange to assign this request to it.
+        exchange = select(request.id).last
+        exchange = build_exchange(request.id) unless exchange&.blank?
+
         # On redirects Chrome doesn't change `requestId` and there's no
         # `Network.responseReceived` event for such request. If there's already
         # exchange object with this id then we got redirected and params has
         # `redirectResponse` key which contains the response.
-        if exchange = first_by(params["requestId"])
-          exchange.build_response(params)
+        if params["redirectResponse"]
+          previous_exchange = select(request.id)[-2]
+          response = Network::Response.new(@page, params)
+          previous_exchange.response = response
         end
 
-        exchange = Network::Exchange.new(@page, params)
-        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
-        @traffic << exchange
+        exchange.request = request
+
+        if exchange.navigation_request?(@page.main_frame.id)
+          @exchange = exchange
+        end
       end
 
       @page.on("Network.responseReceived") do |params|
-        if exchange = last_by(params["requestId"])
-          exchange.build_response(params)
+        if exchange = select(params["requestId"]).last
+          response = Network::Response.new(@page, params)
+          exchange.response = response
         end
       end
 
       @page.on("Network.loadingFinished") do |params|
-        exchange = last_by(params["requestId"])
+        exchange = select(params["requestId"]).last
         if exchange && exchange.response
           exchange.response.body_size = params["encodedDataLength"]
         end
@@ -115,10 +159,10 @@ module Ferrum
 
       @page.on("Log.entryAdded") do |params|
         entry = params["entry"] || {}
-        if entry["source"] == "network" &&
-            entry["level"] == "error" &&
-            exchange = last_by(entry["networkRequestId"])
-          exchange.build_error(entry)
+        if entry["source"] == "network" && entry["level"] == "error"
+          exchange = select(entry["networkRequestId"]).last
+          error = Network::Error.new(entry)
+          exchange.error = error
         end
       end
     end
@@ -135,12 +179,12 @@ module Ferrum
       end
     end
 
-    def first_by(request_id)
-      @traffic.find { |e| e.request.id == request_id }
+    def select(request_id)
+      @traffic.select { |e| e.id == request_id }
     end
 
-    def last_by(request_id)
-      @traffic.select { |e| e.request.id == request_id }.last
+    def build_exchange(id)
+      Network::Exchange.new(@page, id).tap { |e| @traffic << e }
     end
   end
 end

data/lib/ferrum/network/exchange.rb

@@ -1,39 +1,37 @@
 # frozen_string_literal: true
 
-require "ferrum/network/error"
-require "ferrum/network/request"
-require "ferrum/network/response"
-
 module Ferrum
   class Network
     class Exchange
-      attr_reader :request, :response, :error
+      attr_reader :id
+      attr_accessor :intercepted_request
+      attr_accessor :request, :response, :error
 
-      def initialize(page, params)
-        @page = page
-        @response = @error = nil
-        build_request(params)
+      def initialize(page, id)
+        @page, @id = page, id
+        @intercepted_request = nil
+        @request = @response = @error = nil
       end
 
-      def build_request(params)
-        @request = Network::Request.new(params)
+      def navigation_request?(frame_id)
+        request.type?(:document) &&
+          request.frame_id == frame_id
       end
 
-      def build_response(params)
-        @response = Network::Response.new(@page, params)
+      def blank?
+        !request
       end
 
-      def build_error(params)
-        @error = Network::Error.new(params)
+      def blocked?
+        intercepted_request && intercepted_request.status?(:aborted)
       end
 
-      def navigation_request?(frame_id)
-        request.type?(:document) &&
-          request.frame_id == frame_id
+      def finished?
+        blocked? || response || error
       end
 
-      def blocked?
-        response.nil?
+      def pending?
+        !finished?
       end
 
       def to_a
@@ -41,7 +39,12 @@ module Ferrum
       end
 
       def inspect
-        %(#<#{self.class} @id=#{@id.inspect} @request=#{@request.inspect} @response=#{@response.inspect} @error=#{@error.inspect}>)
+        "#<#{self.class} "\
+        "@id=#{@id.inspect} "\
+        "@intercepted_request=#{@intercepted_request.inspect} "\
+        "@request=#{@request.inspect} "\
+        "@response=#{@response.inspect} "\
+        "@error=#{@error.inspect}>"
       end
     end
   end

data/lib/ferrum/network/intercepted_request.rb

@@ -5,14 +5,20 @@ require "base64"
 module Ferrum
   class Network
     class InterceptedRequest
-      attr_accessor :request_id, :frame_id, :resource_type
+      attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
 
       def initialize(page, params)
+        @status = nil
         @page, @params = page, params
         @request_id = params["requestId"]
         @frame_id = params["frameId"]
         @resource_type = params["resourceType"]
         @request = params["request"]
+        @network_id = params["networkId"]
+      end
+
+      def status?(value)
+        @status == value.to_sym
       end
 
       def navigation_request?
@@ -25,7 +31,7 @@ module Ferrum
 
       def respond(**options)
         has_body = options.has_key?(:body)
-        headers = has_body ? { "content-length" => options.fetch(:body, '').length } : {}
+        headers = has_body ? { "content-length" => options.fetch(:body, "").length } : {}
         headers = headers.merge(options.fetch(:responseHeaders, {}))
 
         options = {responseCode: 200}.merge(options)
@@ -33,17 +39,20 @@ module Ferrum
           requestId: request_id,
           responseHeaders: header_array(headers),
         })
-        options = options.merge(body: Base64.encode64(options.fetch(:body, '')).strip) if has_body
+        options = options.merge(body: Base64.encode64(options.fetch(:body, "")).strip) if has_body
 
+        @status = :responded
         @page.command("Fetch.fulfillRequest", **options)
       end
 
       def continue(**options)
         options = options.merge(requestId: request_id)
+        @status = :continued
         @page.command("Fetch.continueRequest", **options)
       end
 
       def abort
+        @status = :aborted
         @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
       end
 

data/lib/ferrum/network/response.rb

@@ -34,6 +34,10 @@ module Ferrum
       def headers_size
         @response["encodedDataLength"]
       end
+      
+      def type
+        @params["type"]
+      end
 
       def content_type
         @content_type ||= headers.find { |k, _| k.downcase == "content-type" }&.last&.sub(/;.*\z/, "")

data/lib/ferrum/page.rb

@@ -13,6 +13,8 @@ require "ferrum/browser/client"
 
 module Ferrum
   class Page
+    GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
+
     class Event < Concurrent::Event
       def iteration
         synchronize { @iteration }
@@ -65,7 +67,7 @@ module Ferrum
     def goto(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
-      response = command("Page.navigate", wait: timeout, **options)
+      response = command("Page.navigate", wait: GOTO_WAIT, **options)
       # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
       if %w[net::ERR_NAME_NOT_RESOLVED
             net::ERR_NAME_RESOLUTION_FAILED
@@ -74,6 +76,9 @@ module Ferrum
         raise StatusError, options[:url]
       end
       response["frameId"]
+    rescue TimeoutError
+      pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
+      raise StatusError.new(options[:url], pendings) unless pendings.empty?
     end
 
     def close
@@ -104,6 +109,7 @@ module Ferrum
     def refresh
       command("Page.reload", wait: timeout)
     end
+    alias_method :reload, :refresh
 
     def back
       history_navigate(delta: -1)
@@ -113,12 +119,23 @@ module Ferrum
       history_navigate(delta: 1)
     end
 
+    def bypass_csp(value = true)
+      enabled = !!value
+      command("Page.setBypassCSP", enabled: enabled)
+      enabled
+    end
+
     def command(method, wait: 0, **params)
       iteration = @event.reset if wait > 0
       result = @client.command(method, params)
       if wait > 0
-        @event.wait(wait)
-        @event.wait(@browser.timeout) if iteration != @event.iteration
+        @event.wait(wait) # Wait a bit after command and check if iteration has
+                          # changed which means there was some network event for
+                          # the main frame and it started to load new content.
+        if iteration != @event.iteration
+          set = @event.wait(@browser.timeout)
+          raise TimeoutError unless set
+        end
       end
       result
     end
@@ -133,6 +150,9 @@ module Ferrum
       when :request
         @client.on("Fetch.requestPaused") do |params, index, total|
           request = Network::InterceptedRequest.new(self, params)
+          exchange = network.select(request.network_id).last
+          exchange ||= network.build_exchange(request.network_id)
+          exchange.intercepted_request = request
           block.call(request, index, total)
         end
       when :auth
@@ -163,14 +183,6 @@ module Ferrum
           Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
         end
       end
-
-      on("Page.domContentEventFired") do |params|
-        # `frameStoppedLoading` doesn't occur if status isn't success
-        if network.status != 200
-          @event.set
-          get_document_id
-        end
-      end
     end
 
     def prepare_page

data/lib/ferrum/page/frames.rb

@@ -40,18 +40,6 @@ module Ferrum
           frame.name = name unless name.to_s.empty?
         end
 
-        on("Page.frameScheduledNavigation") do |params|
-          frame = @frames[params["frameId"]]
-          frame.state = :scheduled_navigation
-          @event.reset
-        end
-
-        on("Page.frameClearedScheduledNavigation") do |params|
-          frame = @frames[params["frameId"]]
-          frame.state = :cleared_scheduled_navigation
-          @event.set if idling?
-        end
-
         on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the

data/lib/ferrum/page/screenshot.rb

@@ -136,7 +136,7 @@ module Ferrum
 
       def capture_screenshot(options, full)
         maybe_resize_fullscreen(full) do
-          command("Page.captureScreenshot", options)
+          command("Page.captureScreenshot", **options)
         end.fetch("data")
       end
 

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.6.2"
+  VERSION = "0.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: '0.7'
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-30 00:00:00.000000000 Z
+date: 2020-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: websocket-driver
@@ -181,7 +181,10 @@ files:
 - README.md
 - lib/ferrum.rb
 - lib/ferrum/browser.rb
+- lib/ferrum/browser/chrome.rb
 - lib/ferrum/browser/client.rb
+- lib/ferrum/browser/command.rb
+- lib/ferrum/browser/firefox.rb
 - lib/ferrum/browser/process.rb
 - lib/ferrum/browser/subscriber.rb
 - lib/ferrum/browser/web_socket.rb