ferrum 0.17.2 → 0.18.0

data/lib/ferrum/network.rb

@@ -8,6 +8,13 @@ require "ferrum/network/request"
 require "ferrum/network/response"
 
 module Ferrum
+  #
+  # Tracks a page's network activity, exposing it as a list of {#traffic}
+  # {Network::Exchange}s built from the underlying CDP `Network.*` events.
+  # Also provides request interception/authorization (`intercept`,
+  # `authorize`, `blacklist=`/`whitelist=`) and network condition emulation
+  # (`emulate_network_conditions`, `offline_mode`).
+  #
   class Network
     CLEAR_TYPE = %i[traffic cache].freeze
     AUTHORIZE_TYPE = %i[server proxy].freeze
@@ -38,6 +45,7 @@ module Ferrum
       @exchange = nil
       @blacklist = nil
       @whitelist = nil
+      @mutex = Mutex.new
     end
 
     #
@@ -81,18 +89,45 @@ module Ferrum
       raise TimeoutError unless result
     end
 
+    #
+    # Whether the network is idle, i.e. no more than `connections`
+    # connections are still pending.
+    #
+    # @param [Integer] connections
+    #   How many connections are allowed for network to be idling.
+    #
+    # @return [Boolean]
+    #
     def idle?(connections = 0)
       pending_connections <= connections
     end
 
+    #
+    # Total number of network connections seen since the traffic was last
+    # cleared.
+    #
+    # @return [Integer]
+    #
     def total_connections
       @traffic.size
     end
 
+    #
+    # Number of network connections that have finished, i.e. were blocked, got
+    # a loaded response, errored, or are otherwise no longer pending.
+    #
+    # @return [Integer]
+    #
     def finished_connections
       @traffic.count(&:finished?)
     end
 
+    #
+    # Number of network connections that are still pending, i.e. haven't
+    # finished yet.
+    #
+    # @return [Integer]
+    #
     def pending_connections
       total_connections - finished_connections
     end
@@ -164,12 +199,38 @@ module Ferrum
       true
     end
 
+    #
+    # Sets a list of patterns for URLs that should be blocked from loading.
+    # Aborts any request whose URL matches one of the given patterns, and
+    # continues all others. Can't be used together with `whitelist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.blacklist = /jquery/
+    #   browser.go_to("https://example.com/")
+    #
     def blacklist=(patterns)
       @blacklist = Array(patterns)
       blacklist_subscribe
     end
     alias blocklist= blacklist=
 
+    #
+    # Sets a list of patterns for URLs that are the only ones allowed to load.
+    # Continues any request whose URL matches one of the given patterns, and
+    # aborts all others. Can't be used together with `blacklist=`.
+    #
+    # @param [String, Regexp, Array<String, Regexp>] patterns
+    #   One or more patterns matched against the request's URL, see
+    #   {InterceptedRequest#match?}.
+    #
+    # @example
+    #   browser.network.whitelist = /example/
+    #   browser.go_to("https://example.com/")
+    #
     def whitelist=(patterns)
       @whitelist = Array(patterns)
       whitelist_subscribe
@@ -268,6 +329,11 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to the CDP events needed to keep track of `traffic`. Called
+    # once when the page is initialized.
+    #
+    # @api private
     def subscribe
       subscribe_request_will_be_sent
       subscribe_response_received
@@ -276,6 +342,23 @@ module Ferrum
       subscribe_log_entry_added
     end
 
+    #
+    # Builds the `authChallengeResponse` sent back to Chrome for an
+    # authenticated request, used by `authorize`.
+    #
+    # @param [Array<String>] ids
+    #   Request ids that were already given credentials, to avoid an infinite
+    #   retry loop if the credentials are rejected.
+    #
+    # @param [String] request_id
+    #
+    # @param [String, nil] username
+    #
+    # @param [String, nil] password
+    #
+    # @return [Hash, nil]
+    #
+    # @api private
     def authorized_response(ids, request_id, username, password)
       if ids.include?(request_id)
         { response: "CancelAuth" }
@@ -286,14 +369,42 @@ module Ferrum
       end
     end
 
+    #
+    # Finds the exchanges in `traffic` with the given request id.
+    #
+    # @param [String] request_id
+    #
+    # @return [Array<Exchange>]
+    #
+    # @api private
     def select(request_id)
       @traffic.select { |e| e.id == request_id }
     end
 
+    #
+    # Builds a new {Exchange} for the given request id and appends it to
+    # `traffic`.
+    #
+    # @param [String] id
+    #
+    # @return [Exchange]
+    #
+    # @api private
     def build_exchange(id)
       Network::Exchange.new(@page, id).tap { |e| @traffic << e }
     end
 
+    # `Network.requestWillBeSent` and `Fetch.requestPaused` are handled on
+    # separate threads (see `Client::Subscriber`), so the "find the existing
+    # exchange for this id or build a new one" check has to be atomic,
+    # otherwise both threads can race past the `select` before either has
+    # appended, and end up building two exchanges for the same request.
+    #
+    # @api private
+    def find_or_build_exchange(id)
+      @mutex.synchronize { select(id).last || build_exchange(id) }
+    end
+
     #
     # Activates emulation of network conditions.
     #
@@ -380,24 +491,25 @@ module Ferrum
 
         # We can build exchange in two places, here on the event or when request
         # is interrupted. So we have to be careful when to create new one. We
-        # create new exchange only if there's no with such id or there's, but
-        # it's filled with request which means this one is new but has response
-        # for a redirect. So we assign response from the params to previous
-        # exchange and build new exchange to assign this request to it.
-        exchange = select(request.id).last
-        exchange = build_exchange(request.id) if exchange.nil? || !exchange.blank?
+        # create a new exchange only if there's no with such an id or there's, but
+        # it's filled with request which means this one is new but has a response
+        # for a redirect. So we assign a response from the params to the previous
+        # exchange and build a new exchange to assign this request to it.
+        exchange = @mutex.synchronize do
+          ex = select(request.id).last
+          ex.nil? || !ex.blank? ? build_exchange(request.id) : ex
+        end
         request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame&.id)
           @exchange = exchange
           classify_pending_exchanges(exchange.loader_id)
         end
       end
 
       @page.on("Network.requestWillBeSentExtraInfo") do |params|
-        exchange = select(params["requestId"]).last
-        exchange ||= build_exchange(params["requestId"])
+        exchange = find_or_build_exchange(params["requestId"])
         exchange.request_extra_info = params
         exchange.request&.headers&.merge!(params["headers"])
       end

data/lib/ferrum/node.rb

@@ -1,36 +1,92 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents a DOM node (an element or a text node) found on a {Page} or
+  # within a {Frame}. Provides methods to inspect it (`text`, `property`,
+  # `attribute`), interact with it (`click`, `focus`, `type`, `select`) and
+  # search within it (`at_css`, `at_xpath`, `css`, `xpath`).
+  #
+  # @note Node identity is tied to the target it was found on; a `Node`
+  #   fetched before a navigation cannot be used afterwards.
+  #
   class Node
     MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
     MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
 
-    attr_reader :page, :target_id, :node_id, :description, :tag_name
+    attr_reader :page, :target_id, :description, :tag_name
 
-    def initialize(frame, target_id, node_id, description)
+    def initialize(frame, target_id, description, object_id: nil, node_id: nil)
       @page = frame.page
       @target_id = target_id
-      @node_id = node_id
       @description = description
       @tag_name = description["nodeName"].downcase
+      @object_id = object_id
+      @node_id = node_id
+    end
+
+    # Frontend node id is resolved lazily, on first actual need (focus, click, scroll_into_view, etc.)
+    # We can try to subscribe to `DOM.childNodeRemoved` and `DOM.childNodeInserted` in the future
+    # to keep track of nodes.
+    def node_id
+      @node_id ||= begin
+        id = page.command("DOM.requestNode", objectId: @object_id)["nodeId"]
+        raise NodeNotFoundError, "node is not trackable" if id.zero?
+
+        id
+      rescue NoExecutionContextError
+        raise NodeNotFoundError, "node is not trackable"
+      end
     end
 
+    # Whether this is an element node, as opposed to e.g. a text node.
+    #
+    # @return [Boolean]
     def node?
       description["nodeType"] == 1 # nodeType: 3, nodeName: "#text" e.g.
     end
 
+    #
+    # The id of the frame this node belongs to.
+    #
+    # @return [String]
+    #
     def frame_id
       description["frameId"]
     end
 
+    #
+    # The {Frame} this node belongs to. Keep using finder methods (`at_css`,
+    # `at_xpath`, etc.) on it to search within that frame, e.g. inside an
+    # `iframe`.
+    #
+    # @return [Frame, nil]
+    #
+    # @example
+    #   frame = page.at_xpath("//iframe").frame # => Frame
+    #   frame.at_css("//a[text() = 'Log in']") # => Node
+    #
     def frame
       page.frame_by(id: frame_id)
     end
 
+    #
+    # Focuses the node.
+    #
+    # @return [self]
+    #
+    # @example
+    #   input = page.at_css("input[name='q']")
+    #   input.focus
+    #
     def focus
       tap { page.command("DOM.focus", slowmoable: true, nodeId: node_id) }
     end
 
+    # Whether the node can receive focus. Attempts to {#focus} the node to
+    # find out.
+    #
+    # @return [Boolean]
     def focusable?
       focus
       true
@@ -38,6 +94,20 @@ module Ferrum
       e.message == "Element is not focusable" ? false : raise
     end
 
+    #
+    # Waits until the node's position stops changing, retrying up to
+    # `attempts` times. Raises {NodeMovingError} if the node is still moving
+    # after the last attempt.
+    #
+    # @param [Float] delay
+    #   Seconds to wait between two position checks.
+    #
+    # @param [Integer] attempts
+    #   Maximum number of attempts before raising.
+    #
+    # @return [Array]
+    #   The content quads of the node once it has stopped moving.
+    #
     def wait_for_stop_moving(delay: MOVING_WAIT_DELAY, attempts: MOVING_WAIT_ATTEMPTS)
       Utils::Attempt.with_retry(errors: NodeMovingError, max: attempts, wait: 0) do
         previous, current = content_quads_with(delay: delay)
@@ -47,15 +117,39 @@ module Ferrum
       end
     end
 
+    # Checks whether the node's position has stopped changing, by comparing
+    # two content-quad snapshots taken `delay` seconds apart.
+    #
+    # @param [Float] delay
+    #   Seconds to wait between the two position checks.
+    #
+    # @return [Boolean]
     def moving?(delay: MOVING_WAIT_DELAY)
       previous, current = content_quads_with(delay: delay)
       previous == current
     end
 
+    #
+    # Removes focus from the node.
+    #
+    # @return [self]
+    #
     def blur
       tap { evaluate("this.blur()") }
     end
 
+    #
+    # Sends keystrokes to the currently focused element via the page's
+    # keyboard. Typically chained after {#focus} or `click`.
+    #
+    # @param [Array<String, Symbol, (Symbol, String)>] keys
+    #   The keys to type, e.g. `"Input"`, `[:Shift, "s"], "tring"`.
+    #
+    # @return [self]
+    #
+    # @example
+    #   input.focus.type("Input")
+    #
     def type(*keys)
       tap { page.keyboard.type(*keys) }
     end
@@ -67,32 +161,46 @@ module Ferrum
       x, y = find_position(**offset)
       modifiers = page.keyboard.modifiers(keys)
 
+      # `:right` and `:double` pass `wait: 0` to preserve the historical
+      # no-network-wait default of `Mouse#up` and `Mouse#down`
       case mode
       when :right
-        page.mouse.move(x: x, y: y)
-        page.mouse.down(button: :right, modifiers: modifiers)
-        sleep(delay)
-        page.mouse.up(button: :right, modifiers: modifiers)
+        page.mouse.click(x:, y:, modifiers:, delay:, button: :right, wait: 0)
       when :double
-        page.mouse.move(x: x, y: y)
-        page.mouse.down(modifiers: modifiers, count: 2)
-        sleep(delay)
-        page.mouse.up(modifiers: modifiers, count: 2)
+        page.mouse.click(x:, y:, modifiers:, delay:, count: 2, wait: 0)
       when :left
-        page.mouse.click(x: x, y: y, modifiers: modifiers, delay: delay)
+        page.mouse.click(x:, y:, modifiers:, delay:)
       end
 
       self
     end
 
+    # Not currently implemented.
+    #
+    # @raise [NotImplementedError] always
     def hover
       raise NotImplementedError
     end
 
+    #
+    # Scrolls the node into view if it is not already visible.
+    #
+    # @return [self]
+    #
+    # @example
+    #   page.at_css("#footer").scroll_into_view
+    #
     def scroll_into_view
       tap { page.command("DOM.scrollIntoViewIfNeeded", nodeId: node_id) }
     end
 
+    # Whether the node's bounding rect is fully within the viewport (or,
+    # when `of:` is given, within that scoping element's bounds).
+    #
+    # @param [Node, nil] of
+    #   An element to use as the visible bounds instead of the window.
+    #
+    # @return [Boolean]
     def in_viewport?(of: nil)
       function = <<~JS
         function(element, scope) {
@@ -109,26 +217,94 @@ module Ferrum
       page.evaluate_func(function, self, of)
     end
 
+    #
+    # Sets files on a file input node.
+    #
+    # @param [String, Array<String>] value
+    #   Path or paths to the file(s) to upload.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.at_css("input[type=file]").select_file("/path/to/file.png")
+    #
     def select_file(value)
-      page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
-    end
-
+      page.command(
+        "DOM.setFileInputFiles",
+        slowmoable: true,
+        backendNodeId: description["backendNodeId"],
+        files: Array(value)
+      )
+    end
+
+    #
+    # Finds a node by xpath, scoped to search within this node. Runs
+    # `document.evaluate` within this node.
+    #
+    # @param [String] selector
+    #
+    # @return [Node, nil]
+    #
+    # @example
+    #   page.at_xpath("//iframe").at_xpath(".//a") # => Node
+    #
     def at_xpath(selector)
       page.at_xpath(selector, within: self)
     end
 
+    #
+    # Finds a node by CSS selector, scoped to search within this node. Runs
+    # `querySelector` within this node.
+    #
+    # @param [String] selector
+    #
+    # @return [Node, nil]
+    #
+    # @example
+    #   page.at_css("form").at_css("input[name='q']") # => Node
+    #
     def at_css(selector)
       page.at_css(selector, within: self)
     end
 
+    #
+    # Finds nodes by xpath, scoped to search within this node. Runs
+    # `document.evaluate` within this node.
+    #
+    # @param [String] selector
+    #
+    # @return [Array<Node>]
+    #
+    # @example
+    #   page.at_css("ul").xpath(".//li") # => [Node]
+    #
     def xpath(selector)
       page.xpath(selector, within: self)
     end
 
+    #
+    # Finds nodes by CSS selector, scoped to search within this node. Runs
+    # `querySelectorAll` within this node.
+    #
+    # @param [String] selector
+    #
+    # @return [Array<Node>]
+    #
+    # @example
+    #   page.at_css("ul").css("li") # => [Node]
+    #
     def css(selector)
       page.css(selector, within: self)
     end
 
+    #
+    # The node's text content, i.e. `textContent`.
+    #
+    # @return [String]
+    #
+    # @example
+    #   page.at_css("a > h3").text # => "rubycdp/ferrum: Ruby Chrome/Chromium driver - GitHub"
+    #
     def text
       evaluate("this.textContent")
     end
@@ -138,19 +314,52 @@ module Ferrum
       evaluate("this.innerText")
     end
 
+    #
+    # The node's `value` property. Useful for form elements such as `input`,
+    # `select` and `textarea`.
+    #
+    # @return [Object]
+    #
     def value
       evaluate("this.value")
     end
 
+    #
+    # Returns the given JavaScript property of the node.
+    #
+    # @param [String] name
+    #
+    # @return [Object]
+    #
+    # @example
+    #   page.at_css("input").property("value") # => "Foo"
+    #
     def property(name)
       evaluate("this['#{name}']")
     end
     alias [] property
 
+    #
+    # Returns the value of the given HTML attribute, i.e.
+    # `getAttribute(name)`. Unlike {#property}, it reads the attribute as
+    # defined in markup rather than the live DOM property.
+    #
+    # @param [String] name
+    #
+    # @return [String, nil]
+    #
+    # @example
+    #   page.at_css("input").attribute("value") # => "Foo"
+    #
     def attribute(name)
       evaluate("this.getAttribute('#{name}')")
     end
 
+    #
+    # Returns the selected `option` nodes of a `select` element.
+    #
+    # @return [Array<Node>]
+    #
     def selected
       function = <<~JS
         function(element) {
@@ -163,6 +372,29 @@ module Ferrum
       page.evaluate_func(function, self, on: self)
     end
 
+    #
+    # (chainable) Selects options of a `select` element by the given
+    # attribute.
+    #
+    # @param [Array<String>] values
+    #   The value(s) to select. Accepts a string, multiple strings, or an
+    #   array of strings.
+    #
+    # @param [Symbol] by
+    #   The `option` attribute to match `values` against, e.g. `:value` or
+    #   `:text`.
+    #
+    # @return [self]
+    #
+    # @example
+    #   page.at_xpath("//*[select]").select(["1"]) # => Node (select)
+    #   page.at_xpath("//*[select]").select(["text"], by: :text) # => Node (select)
+    #
+    # @example Accepts a string, multiple strings or an array of strings:
+    #   page.at_xpath("//*[select]").select("1")
+    #   page.at_xpath("//*[select]").select("1", "2")
+    #   page.at_xpath("//*[select]").select(["1", "2"])
+    #
     def select(*values, by: :value)
       tap do
         function = <<~JS
@@ -184,10 +416,29 @@ module Ferrum
       end
     end
 
+    #
+    # Evaluates the given JavaScript expression with `this` bound to the
+    # node.
+    #
+    # @param [String] expression
+    #
+    # @return [Object]
+    #
+    # @example
+    #   page.at_css("input").evaluate("this.value")
+    #
     def evaluate(expression)
       page.evaluate_on(node: self, expression: expression)
     end
 
+    #
+    # Two nodes are equal when they belong to the same target and share the
+    # same backend node id.
+    #
+    # @param [Object] other
+    #
+    # @return [Boolean]
+    #
     def ==(other)
       return false unless other.is_a?(Node)
 
@@ -197,10 +448,30 @@ module Ferrum
       target_id == other.target_id && description["backendNodeId"] == other.description["backendNodeId"]
     end
 
+    #
+    # A developer-friendly string representation of the node.
+    #
+    # @return [String]
+    #
     def inspect
       %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
     end
 
+    #
+    # Finds the x, y coordinates to click or hover on the node.
+    #
+    # @param [Integer, nil] x
+    #   Horizontal offset from the reference point.
+    #
+    # @param [Integer, nil] y
+    #   Vertical offset from the reference point.
+    #
+    # @param [Symbol] position
+    #   `:top` to offset from the node's top-left corner, `:center` to offset
+    #   from its center.
+    #
+    # @return [(Integer, Integer)]
+    #
     def find_position(x: nil, y: nil, position: :top)
       points = wait_for_stop_moving.map { |q| to_points(q) }.first
       get_position(points, x, y, position)
@@ -218,10 +489,29 @@ module Ferrum
         .each_with_object({}) { |style, memo| memo.merge!(style["name"] => style["value"]) }
     end
 
+    # Returns the computed accessibility node for the element, or nil if the
+    # element is ignored by the accessibility tree.
+    #
+    # @return [Accessibility::AXNode, nil]
+    def axnode
+      page.accessibility.node_for(self)
+    end
+
+    #
+    # Removes the node from the DOM.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.at_css("#ad").remove
+    #
     def remove
       page.command("DOM.removeNode", nodeId: node_id)
     end
 
+    # Whether the node still exists in the DOM.
+    #
+    # @return [Boolean]
     def exists?
       page.command("DOM.resolveNode", nodeId: node_id)
       true

data/lib/ferrum/page/animation.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Page
+    #
+    # Controls the playback of CSS animations on the page via the CDP
+    # [Animation](https://chromedevtools.github.io/devtools-protocol/tot/Animation/)
+    # domain, allowing animations to be sped up, slowed down, or paused.
+    #
     module Animation
       #
       # Returns playback rate for CSS animations, defaults to `1`.