ferrum 0.17.1 → 0.18.0

data/lib/ferrum/network/response.rb

@@ -124,7 +124,9 @@ module Ferrum
       #
       # @return [String]
       #
-      def body
+      # @raise [Ferrum::BrowserError]
+      #
+      def body!
         @body ||= begin
           body, encoded = @page.command("Network.getResponseBody", requestId: id)
                                .values_at("body", "base64Encoded")
@@ -132,6 +134,19 @@ module Ferrum
         end
       end
 
+      #
+      # The response body.
+      #
+      # @return [String, nil]
+      #
+      def body
+        body!
+      rescue Ferrum::BrowserError
+        # nop
+      end
+
+      #
+      # Whether this response is the main frame's current response.
       #
       # @return [Boolean]
       #

data/lib/ferrum/network.rb

@@ -8,6 +8,13 @@ require "ferrum/network/request"
 require "ferrum/network/response"
 
 module Ferrum
+  #
+  # Tracks a page's network activity, exposing it as a list of {#traffic}
+  # {Network::Exchange}s built from the underlying CDP `Network.*` events.
+  # Also provides request interception/authorization (`intercept`,
+  # `authorize`, `blacklist=`/`whitelist=`) and network condition emulation
+  # (`emulate_network_conditions`, `offline_mode`).
+  #
   class Network
     CLEAR_TYPE = %i[traffic cache].freeze
     AUTHORIZE_TYPE = %i[server proxy].freeze
@@ -17,7 +24,7 @@ module Ferrum
                         SignedExchange Ping CSPViolationReport Preflight Other].freeze
     AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
                               "or subscribe to `on(:request)` events before calling it"
-    AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
+    AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}".freeze
     ALLOWED_CONNECTION_TYPE = %w[none cellular2g cellular3g cellular4g bluetooth ethernet wifi wimax other].freeze
 
     # Network traffic.
@@ -38,6 +45,7 @@ module Ferrum
       @exchange = nil
       @blacklist = nil
       @whitelist = nil
+      @mutex = Mutex.new
     end
 
     #
@@ -81,18 +89,45 @@ module Ferrum
       raise TimeoutError unless result
     end
 
+    #
+    # Whether the network is idle, i.e. no more than `connections`
+    # connections are still pending.
+    #
+    # @param [Integer] connections
+    #   How many connections are allowed for network to be idling.
+    #
+    # @return [Boolean]
+    #
     def idle?(connections = 0)
       pending_connections <= connections
     end
 
+    #
+    # Total number of network connections seen since the traffic was last
+    # cleared.
+    #
+    # @return [Integer]
+    #
     def total_connections
       @traffic.size
     end
 
+    #
+    # Number of network connections that have finished, i.e. were blocked, got
+    # a loaded response, errored, or are otherwise no longer pending.
+    #
+    # @return [Integer]
+    #
     def finished_connections
       @traffic.count(&:finished?)
     end
 
+    #
+    # Number of network connections that are still pending, i.e. haven't
+    # finished yet.
+    #
+    # @return [Integer]
+    #
     def pending_connections
       total_connections - finished_connections
     end
@@ -164,12 +199,38 @@ module Ferrum
       true
     end
 
+    #
+    # Sets a list of patterns for URLs that should be blocked from loading.
+    # Aborts any request whose URL matches one of the given patterns, and
+    # continues all others. Can't be used together with `whitelist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.blacklist = /jquery/
+    #   browser.go_to("https://example.com/")
+    #
     def blacklist=(patterns)
       @blacklist = Array(patterns)
       blacklist_subscribe
     end
     alias blocklist= blacklist=
 
+    #
+    # Sets a list of patterns for URLs that are the only ones allowed to load.
+    # Continues any request whose URL matches one of the given patterns, and
+    # aborts all others. Can't be used together with `blacklist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.whitelist = /example/
+    #   browser.go_to("https://example.com/")
+    #
     def whitelist=(patterns)
       @whitelist = Array(patterns)
       whitelist_subscribe
@@ -268,6 +329,11 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to the CDP events needed to keep track of `traffic`. Called
+    # once when the page is initialized.
+    #
+    # @api private
     def subscribe
       subscribe_request_will_be_sent
       subscribe_response_received
@@ -276,6 +342,23 @@ module Ferrum
       subscribe_log_entry_added
     end
 
+    #
+    # Builds the `authChallengeResponse` sent back to Chrome for an
+    # authenticated request, used by `authorize`.
+    #
+    # @param [Array<String>] ids
+    #   Request ids that were already given credentials, to avoid an infinite
+    #   retry loop if the credentials are rejected.
+    #
+    # @param [String] request_id
+    #
+    # @param [String, nil] username
+    #
+    # @param [String, nil] password
+    #
+    # @return [Hash, nil]
+    #
+    # @api private
     def authorized_response(ids, request_id, username, password)
       if ids.include?(request_id)
         { response: "CancelAuth" }
@@ -286,14 +369,42 @@ module Ferrum
       end
     end
 
+    #
+    # Finds the exchanges in `traffic` with the given request id.
+    #
+    # @param [String] request_id
+    #
+    # @return [Array<Exchange>]
+    #
+    # @api private
     def select(request_id)
       @traffic.select { |e| e.id == request_id }
     end
 
+    #
+    # Builds a new {Exchange} for the given request id and appends it to
+    # `traffic`.
+    #
+    # @param [String] id
+    #
+    # @return [Exchange]
+    #
+    # @api private
     def build_exchange(id)
       Network::Exchange.new(@page, id).tap { |e| @traffic << e }
     end
 
+    # `Network.requestWillBeSent` and `Fetch.requestPaused` are handled on
+    # separate threads (see `Client::Subscriber`), so the "find the existing
+    # exchange for this id or build a new one" check has to be atomic,
+    # otherwise both threads can race past the `select` before either has
+    # appended, and end up building two exchanges for the same request.
+    #
+    # @api private
+    def find_or_build_exchange(id)
+      @mutex.synchronize { select(id).last || build_exchange(id) }
+    end
+
     #
     # Activates emulation of network conditions.
     #
@@ -380,24 +491,25 @@ module Ferrum
 
         # We can build exchange in two places, here on the event or when request
         # is interrupted. So we have to be careful when to create new one. We
-        # create new exchange only if there's no with such id or there's, but
-        # it's filled with request which means this one is new but has response
-        # for a redirect. So we assign response from the params to previous
-        # exchange and build new exchange to assign this request to it.
-        exchange = select(request.id).last
-        exchange = build_exchange(request.id) if exchange.nil? || !exchange.blank?
+        # create a new exchange only if there's no with such an id or there's, but
+        # it's filled with request which means this one is new but has a response
+        # for a redirect. So we assign a response from the params to the previous
+        # exchange and build a new exchange to assign this request to it.
+        exchange = @mutex.synchronize do
+          ex = select(request.id).last
+          ex.nil? || !ex.blank? ? build_exchange(request.id) : ex
+        end
         request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame&.id)
           @exchange = exchange
           classify_pending_exchanges(exchange.loader_id)
         end
       end
 
       @page.on("Network.requestWillBeSentExtraInfo") do |params|
-        exchange = select(params["requestId"]).last
-        exchange ||= build_exchange(params["requestId"])
+        exchange = find_or_build_exchange(params["requestId"])
         exchange.request_extra_info = params
         exchange.request&.headers&.merge!(params["headers"])
       end