ferrum 0.17.2 → 0.19.0

data/lib/ferrum/mouse.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Simulates mouse input for a page via the CDP `Input.dispatchMouseEvent`
+  # command, and page scrolling via JavaScript. Keeps track of the current
+  # pointer position and pressed buttons so `move`, `down`, `up` and `click`
+  # can be composed while reporting consistent state to Chrome.
+  #
   class Mouse
     CLICK_WAIT = ENV.fetch("FERRUM_CLICK_WAIT", 0.1).to_f
     BUTTON_MASKS = {

data/lib/ferrum/network/auth_request.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Network
+    #
+    # Represents an HTTP authentication challenge (basic or digest auth
+    # prompt) raised by the browser via `Fetch.authRequired`, which can be
+    # answered with credentials or canceled.
+    #
     class AuthRequest
       attr_accessor :request_id, :frame_id, :resource_type
 
@@ -14,47 +19,121 @@ module Ferrum
         @request = params["request"]
       end
 
+      #
+      # Whether this request is for the navigation of a frame, as opposed
+      # to a subresource (script, image, XHR, etc).
+      #
+      # @return [Boolean]
+      #
       def navigation_request?
         @params["isNavigationRequest"]
       end
 
+      #
+      # Whether the authentication challenge's source matches the given
+      # type.
+      #
+      # @param [String, Symbol] source
+      #   One of `:server` or `:proxy`.
+      #
+      # @return [Boolean]
+      #
       def auth_challenge?(source)
         @params.dig("authChallenge", "source")&.downcase&.to_s == source.to_s
       end
 
-      def match?(regexp)
-        !!url.match(regexp)
+      #
+      # Whether the request's URL matches the given pattern.
+      #
+      # @param [String, Regexp] pattern
+      #
+      # @return [Boolean]
+      #
+      def match?(pattern)
+        case url
+        when pattern
+          true
+        else
+          false
+        end
       end
 
+      #
+      # Responds to the authentication challenge, e.g. with credentials or by
+      # canceling it.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :response
+      #   One of `"Default"`, `"CancelAuth"`, or `"ProvideCredentials"`.
+      #
+      # @option options [String] :username
+      #
+      # @option options [String] :password
+      #
       def continue(**options)
         options = options.merge(requestId: request_id)
         @page.command("Fetch.continueWithAuth", **options)
       end
 
+      #
+      # Cancels the authentication challenge and the request.
+      #
       def abort
         @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
       end
 
+      #
+      # The URL for the request.
+      #
+      # @return [String]
+      #
       def url
         @request["url"]
       end
 
+      #
+      # The request method.
+      #
+      # @return [String]
+      #
       def method
         @request["method"]
       end
 
+      #
+      # The request headers.
+      #
+      # @return [Hash{String => String}]
+      #
       def headers
         @request["headers"]
       end
 
+      #
+      # The request's initial priority, one of `"VeryLow"`, `"Low"`,
+      # `"Medium"`, `"High"`, or `"VeryHigh"`.
+      #
+      # @return [String]
+      #
       def initial_priority
         @request["initialPriority"]
       end
 
+      #
+      # The request's referrer policy.
+      #
+      # @return [String]
+      #
       def referrer_policy
         @request["referrerPolicy"]
       end
 
+      #
+      # Inspects the auth request.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@request_id=#{@request_id.inspect} " \

data/lib/ferrum/network/error.rb

@@ -2,15 +2,30 @@
 
 module Ferrum
   class Network
+    #
+    # Represents a network-level loading error reported for a request,
+    # e.g. via `Network.loadingFailed`. Distinct from the Ruby exceptions
+    # defined in `ferrum/errors.rb`.
+    #
     class Error
       attr_writer :canceled
       attr_reader :time, :timestamp
       attr_accessor :id, :url, :type, :error_text, :monotonic_time, :description
 
+      # Whether the request was canceled.
+      #
+      # @return [Boolean]
       def canceled?
         @canceled
       end
 
+      # Sets the error's timestamp, deriving {#time} from it.
+      #
+      # @param [Float] value
+      #   Timestamp in milliseconds since epoch, as reported by
+      #   `Log.entryAdded`.
+      #
+      # @return [Float]
       def timestamp=(value)
         @timestamp = value
         @time = Time.strptime((value / 1000).to_s, "%s")

data/lib/ferrum/network/exchange.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Network
+    #
+    # Pairs a {Request} with its eventual {Response} or {Error} (and, if
+    # interception is enabled, its {InterceptedRequest}), representing the
+    # full lifecycle of a single network request.
+    #
     class Exchange
       # ID of the request.
       #
@@ -34,6 +39,11 @@ module Ferrum
       # @return Boolean
       attr_accessor :unknown
 
+      # The raw `Network.requestWillBeSentExtraInfo` params for the request,
+      # if it arrived before `request` was set.
+      #
+      # @return [Hash, nil]
+      #
       # @api private
       attr_accessor :request_extra_info
 

data/lib/ferrum/network/intercepted_request.rb

@@ -5,6 +5,11 @@ require "base64"
 
 module Ferrum
   class Network
+    #
+    # Represents a request paused by request interception (`Fetch.enable`),
+    # allowing it to be continued as is or with overrides, fulfilled with a
+    # fake response, or aborted before it reaches the network.
+    #
     class InterceptedRequest
       include RequestParams
 
@@ -21,18 +26,62 @@ module Ferrum
         @network_id = params["networkId"]
       end
 
+      #
+      # Whether the request's current status matches the given value.
+      #
+      # @param [String, Symbol] value
+      #   One of `:responded`, `:continued`, `:aborted`.
+      #
+      # @return [Boolean]
+      #
       def status?(value)
         @status == value.to_sym
       end
 
+      #
+      # Whether this request is for the navigation of a frame, as opposed
+      # to a subresource (script, image, XHR, etc).
+      #
+      # @return [Boolean]
+      #
       def navigation_request?
         @params["isNavigationRequest"]
       end
 
-      def match?(regexp)
-        !!url.match(regexp)
+      #
+      # Whether the request's URL matches the given pattern.
+      #
+      # @param [String, Regexp] pattern
+      #
+      # @return [Boolean]
+      #
+      def match?(pattern)
+        case url
+        when pattern
+          true
+        else
+          false
+        end
       end
 
+      #
+      # Fulfills the intercepted request with a fake response instead of
+      # letting it reach the network.
+      #
+      # @param [Hash] options
+      #
+      # @option options [Integer] :responseCode
+      #   HTTP status code to respond with, `200` by default.
+      #
+      # @option options [String] :body
+      #   Response body.
+      #
+      # @option options [Hash] :responseHeaders
+      #   Response headers.
+      #
+      # @example
+      #   request.respond(body: "Lorem ipsum")
+      #
       def respond(**options)
         has_body = options.key?(:body)
         headers = has_body ? { "content-length" => options.fetch(:body, "").length } : {}
@@ -46,25 +95,68 @@ module Ferrum
         @client.command("Fetch.fulfillRequest", async: true, **options)
       end
 
+      #
+      # Continues the intercepted request, letting it reach the network as
+      # is, or with the given overrides.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :url
+      #   Overrides the request's URL.
+      #
+      # @option options [String] :method
+      #   Overrides the request's method.
+      #
+      # @option options [String] :postData
+      #   Overrides the request's post data.
+      #
+      # @option options [Hash] :headers
+      #   Overrides the request's headers.
+      #
+      # @example
+      #   request.continue
+      #
       def continue(**options)
         options = options.merge(requestId: request_id)
         @status = :continued
         @client.command("Fetch.continueRequest", async: true, **options)
       end
 
+      #
+      # Aborts the intercepted request.
+      #
+      # @example
+      #   request.abort
+      #
       def abort
         @status = :aborted
         @client.command("Fetch.failRequest", async: true, requestId: request_id, errorReason: "BlockedByClient")
       end
 
+      #
+      # The request's initial priority, one of `"VeryLow"`, `"Low"`,
+      # `"Medium"`, `"High"`, or `"VeryHigh"`.
+      #
+      # @return [String]
+      #
       def initial_priority
         @request["initialPriority"]
       end
 
+      #
+      # The request's referrer policy.
+      #
+      # @return [String]
+      #
       def referrer_policy
         @request["referrerPolicy"]
       end
 
+      #
+      # Inspects the intercepted request.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@request_id=#{@request_id.inspect} " \

data/lib/ferrum/network/request.rb

@@ -33,7 +33,7 @@ module Ferrum
       end
 
       #
-      # The request resouce type.
+      # The request resource type.
       #
       # @return [String]
       #

data/lib/ferrum/network/response.rb

@@ -145,6 +145,8 @@ module Ferrum
         # nop
       end
 
+      #
+      # Whether this response is the main frame's current response.
       #
       # @return [Boolean]
       #

data/lib/ferrum/network.rb

@@ -8,6 +8,13 @@ require "ferrum/network/request"
 require "ferrum/network/response"
 
 module Ferrum
+  #
+  # Tracks a page's network activity, exposing it as a list of {#traffic}
+  # {Network::Exchange}s built from the underlying CDP `Network.*` events.
+  # Also provides request interception/authorization (`intercept`,
+  # `authorize`, `blacklist=`/`whitelist=`) and network condition emulation
+  # (`emulate_network_conditions`, `offline_mode`).
+  #
   class Network
     CLEAR_TYPE = %i[traffic cache].freeze
     AUTHORIZE_TYPE = %i[server proxy].freeze
@@ -38,6 +45,7 @@ module Ferrum
       @exchange = nil
       @blacklist = nil
       @whitelist = nil
+      @mutex = Mutex.new
     end
 
     #
@@ -81,18 +89,45 @@ module Ferrum
       raise TimeoutError unless result
     end
 
+    #
+    # Whether the network is idle, i.e. no more than `connections`
+    # connections are still pending.
+    #
+    # @param [Integer] connections
+    #   How many connections are allowed for network to be idling.
+    #
+    # @return [Boolean]
+    #
     def idle?(connections = 0)
       pending_connections <= connections
     end
 
+    #
+    # Total number of network connections seen since the traffic was last
+    # cleared.
+    #
+    # @return [Integer]
+    #
     def total_connections
       @traffic.size
     end
 
+    #
+    # Number of network connections that have finished, i.e. were blocked, got
+    # a loaded response, errored, or are otherwise no longer pending.
+    #
+    # @return [Integer]
+    #
     def finished_connections
       @traffic.count(&:finished?)
     end
 
+    #
+    # Number of network connections that are still pending, i.e. haven't
+    # finished yet.
+    #
+    # @return [Integer]
+    #
     def pending_connections
       total_connections - finished_connections
     end
@@ -164,12 +199,38 @@ module Ferrum
       true
     end
 
+    #
+    # Sets a list of patterns for URLs that should be blocked from loading.
+    # Aborts any request whose URL matches one of the given patterns, and
+    # continues all others. Can't be used together with `whitelist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.blacklist = /jquery/
+    #   browser.go_to("https://example.com/")
+    #
     def blacklist=(patterns)
       @blacklist = Array(patterns)
       blacklist_subscribe
     end
     alias blocklist= blacklist=
 
+    #
+    # Sets a list of patterns for URLs that are the only ones allowed to load.
+    # Continues any request whose URL matches one of the given patterns, and
+    # aborts all others. Can't be used together with `blacklist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.whitelist = /example/
+    #   browser.go_to("https://example.com/")
+    #
     def whitelist=(patterns)
       @whitelist = Array(patterns)
       whitelist_subscribe
@@ -268,6 +329,11 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to the CDP events needed to keep track of `traffic`. Called
+    # once when the page is initialized.
+    #
+    # @api private
     def subscribe
       subscribe_request_will_be_sent
       subscribe_response_received
@@ -276,6 +342,23 @@ module Ferrum
       subscribe_log_entry_added
     end
 
+    #
+    # Builds the `authChallengeResponse` sent back to Chrome for an
+    # authenticated request, used by `authorize`.
+    #
+    # @param [Array<String>] ids
+    #   Request ids that were already given credentials, to avoid an infinite
+    #   retry loop if the credentials are rejected.
+    #
+    # @param [String] request_id
+    #
+    # @param [String, nil] username
+    #
+    # @param [String, nil] password
+    #
+    # @return [Hash, nil]
+    #
+    # @api private
     def authorized_response(ids, request_id, username, password)
       if ids.include?(request_id)
         { response: "CancelAuth" }
@@ -286,14 +369,42 @@ module Ferrum
       end
     end
 
+    #
+    # Finds the exchanges in `traffic` with the given request id.
+    #
+    # @param [String] request_id
+    #
+    # @return [Array<Exchange>]
+    #
+    # @api private
     def select(request_id)
       @traffic.select { |e| e.id == request_id }
     end
 
+    #
+    # Builds a new {Exchange} for the given request id and appends it to
+    # `traffic`.
+    #
+    # @param [String] id
+    #
+    # @return [Exchange]
+    #
+    # @api private
     def build_exchange(id)
       Network::Exchange.new(@page, id).tap { |e| @traffic << e }
     end
 
+    # `Network.requestWillBeSent` and `Fetch.requestPaused` are handled on
+    # separate threads (see `Client::Subscriber`), so the "find the existing
+    # exchange for this id or build a new one" check has to be atomic,
+    # otherwise both threads can race past the `select` before either has
+    # appended, and end up building two exchanges for the same request.
+    #
+    # @api private
+    def find_or_build_exchange(id)
+      @mutex.synchronize { select(id).last || build_exchange(id) }
+    end
+
     #
     # Activates emulation of network conditions.
     #
@@ -380,24 +491,25 @@ module Ferrum
 
         # We can build exchange in two places, here on the event or when request
         # is interrupted. So we have to be careful when to create new one. We
-        # create new exchange only if there's no with such id or there's, but
-        # it's filled with request which means this one is new but has response
-        # for a redirect. So we assign response from the params to previous
-        # exchange and build new exchange to assign this request to it.
-        exchange = select(request.id).last
-        exchange = build_exchange(request.id) if exchange.nil? || !exchange.blank?
+        # create a new exchange only if there's no with such an id or there's, but
+        # it's filled with request which means this one is new but has a response
+        # for a redirect. So we assign a response from the params to the previous
+        # exchange and build a new exchange to assign this request to it.
+        exchange = @mutex.synchronize do
+          ex = select(request.id).last
+          ex.nil? || !ex.blank? ? build_exchange(request.id) : ex
+        end
         request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame&.id)
           @exchange = exchange
           classify_pending_exchanges(exchange.loader_id)
         end
       end
 
       @page.on("Network.requestWillBeSentExtraInfo") do |params|
-        exchange = select(params["requestId"]).last
-        exchange ||= build_exchange(params["requestId"])
+        exchange = find_or_build_exchange(params["requestId"])
         exchange.request_extra_info = params
         exchange.request&.headers&.merge!(params["headers"])
       end