ferrum 0.17.2 → 0.19.0

data/lib/ferrum/errors.rb

@@ -1,47 +1,105 @@
 # frozen_string_literal: true
 
 module Ferrum
+  # Base class for all errors raised by Ferrum.
   class Error               < StandardError; end
+  # Raised when the referenced page no longer exists.
   class NoSuchPageError     < Error; end
+  # Raised when the referenced browser target cannot be found.
   class NoSuchTargetError   < Error; end
+  # Raised when a requested feature isn't implemented for the current setup.
   class NotImplementedError < Error; end
+  # Raised when no browser binary can be found at the configured path.
   class BinaryNotFoundError < Error; end
+  # Raised when a required file path option is empty.
   class EmptyPathError      < Error; end
+  # Raised when the browser process server fails to start or crashes.
   class ServerError         < Error; end
 
+  # Raised when a request fails to reach the server, e.g. due to DNS or
+  # connectivity issues.
   class StatusError < Error
     def initialize(url, message = nil)
       super(message || "Request to #{url} failed to reach server, check DNS and server status")
     end
   end
 
+  # Raised when the request reached the server, but the page still has
+  # pending network connections that never settled.
   class PendingConnectionsError < StatusError
     attr_reader :pendings
 
     def initialize(url, pendings = [])
       @pendings = pendings
-
-      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+      message = "Request to #{url} reached server, but there are still pending connections"
+      message += ": #{pendings.join(', ')}" unless @pendings.empty?
 
       super(url, message)
     end
   end
 
+  # Raised when waiting for a response from the browser times out.
   class TimeoutError < Error
+    # The CDP method that didn't answer in time, `nil` when the wait wasn't for a command.
+    #
+    # @return [String, nil]
+    attr_reader :command
+
+    # The session the command was sent to, `nil` for the browser-wide session.
+    #
+    # @return [String, nil]
+    attr_reader :session_id
+
+    #
+    # @param [String, nil] command
+    #   The CDP method that timed out.
+    #
+    # @param [String, nil] session_id
+    #   The session it was sent to.
+    #
+    def initialize(command = nil, session_id: nil)
+      @command = command
+      @session_id = session_id
+      super()
+    end
+
+    #
+    # Explains that waiting for a response timed out, naming the command when
+    # there was one.
+    #
+    # @return [String]
+    #
     def message
-      "Timed out waiting for response. It's possible that this happened " \
+      "Timed out waiting for #{awaited}. It's possible that this happened " \
         "because something took a very long time (for example a page load " \
         "was slow). If so, setting the :timeout option to a higher value might " \
-        "help."
+        "help. If this happened on an internal protocol call instead, try " \
+        "raising :protocol_timeout."
+    end
+
+    private
+
+    def awaited
+      return "response" unless @command
+
+      @session_id ? "a response to #{@command} (session #{@session_id})" : "a response to #{@command}"
     end
   end
 
+  # Raised when an evaluated script takes too long to return a value.
   class ScriptTimeoutError < Error
+    #
+    # Explains that the evaluated script timed out.
+    #
+    # @return [String]
+    #
     def message
       "Timed out waiting for evaluated script to return a value"
     end
   end
 
+  # Raised when the browser process doesn't produce a websocket URL within
+  # the configured `:process_timeout`.
   class ProcessTimeoutError < Error
     attr_reader :output
 
@@ -51,12 +109,28 @@ module Ferrum
     end
   end
 
+  # Raised when connecting to an already running browser, but its
+  # `/json/version` endpoint doesn't answer with the websocket url.
+  class NoWebSocketUrlError < Error
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      super("Couldn't get a websocket url out of #{url}, `/json/version` there didn't answer with JSON. " \
+            "Check the browser is reachable at that url and that any credentials it needs are part of it")
+    end
+  end
+
+  # Raised when trying to interact with a browser process that has died or
+  # a window that has already been closed.
   class DeadBrowserError < Error
     def initialize(message = "Browser is dead or given window is closed")
       super
     end
   end
 
+  # Raised when a node keeps moving between attempts to interact with it,
+  # e.g. while trying to click it.
   class NodeMovingError < Error
     def initialize(node, prev, current)
       @node = node
@@ -65,6 +139,11 @@ module Ferrum
       super(message)
     end
 
+    #
+    # Explains that the node moved between attempts to interact with it.
+    #
+    # @return [String]
+    #
     def message
       "#{@node.inspect} that you're trying to click is moving, hence " \
         "we cannot. Previously it was at #{@prev.inspect} but now at " \
@@ -72,12 +151,15 @@ module Ferrum
     end
   end
 
+  # Raised when the content quads (coordinates) for a node cannot be computed.
   class CoordinatesNotFoundError < Error
     def initialize(message = "Could not compute content quads")
       super
     end
   end
 
+  # Raised when the `:format` option passed to a screenshot call is not one
+  # of the supported formats.
   class InvalidScreenshotFormatError < Error
     def initialize(format)
       valid_formats = Page::Screenshot::SUPPORTED_SCREENSHOT_FORMAT.join(" | ")
@@ -85,6 +167,8 @@ module Ferrum
     end
   end
 
+  # Raised when the browser returns an error response over CDP. Wraps the
+  # raw response so callers can inspect its code and data.
   class BrowserError < Error
     attr_reader :response
 
@@ -93,23 +177,38 @@ module Ferrum
       super(response["message"])
     end
 
+    #
+    # Error code from the raw CDP error response.
+    #
+    # @return [Integer, nil]
+    #
     def code
       response["code"]
     end
 
+    #
+    # Additional data from the raw CDP error response.
+    #
+    # @return [Object, nil]
+    #
     def data
       response["data"]
     end
   end
 
+  # Raised when the browser reports that a DOM node no longer exists.
   class NodeNotFoundError < BrowserError; end
 
+  # Raised when there's no JavaScript execution context available to run a
+  # command against, e.g. because the frame isn't attached.
   class NoExecutionContextError < BrowserError
     def initialize(response = nil)
       super(response || { "message" => "There's no context available" })
     end
   end
 
+  # Raised when JavaScript evaluated in the page throws an exception. Carries
+  # the JS exception's class name, message, and stack trace.
   class JavaScriptError < BrowserError
     attr_reader :class_name, :message, :stack_trace
 

data/lib/ferrum/frame/dom.rb

@@ -19,6 +19,12 @@
 # details (DOM.describeNode).
 module Ferrum
   class Frame
+    #
+    # Evaluates and executes JavaScript to query and manipulate a frame's
+    # DOM: reading the URL, title, doctype, and HTML, finding nodes by
+    # XPath or CSS selector, and injecting `<script>`, `<style>`, and
+    # `<link>` tags.
+    #
     module DOM
       SCRIPT_SRC_TAG = <<~JS
         const script = document.createElement("script");
@@ -77,6 +83,17 @@ module Ferrum
         evaluate("window.top.document.title")
       end
 
+      #
+      # Returns current document's doctype declaration.
+      #
+      # @return [String, nil]
+      #   The serialized `<!DOCTYPE ...>` declaration, or +nil+ if the
+      #   document has none.
+      #
+      # @example
+      #   browser.go_to("https://example.com")
+      #   browser.doctype # => "<!DOCTYPE html>"
+      #
       def doctype
         evaluate("document.doctype && new XMLSerializer().serializeToString(document.doctype)")
       end
@@ -226,6 +243,54 @@ module Ferrum
         evaluate_func(expr, selector, within)
       end
 
+      #
+      # Finds a node by using a CSS path or XPath selector, retrying until
+      # it appears or `timeout` elapses. Pass exactly one of `css`/`xpath`;
+      # each attempt delegates to {#at_css}/{#at_xpath} respectively, so it
+      # matches their behavior exactly.
+      #
+      # @param [String, nil] css
+      #   The CSS path selector. Mutually exclusive with `xpath`.
+      #
+      # @param [String, nil] xpath
+      #   The XPath selector. Mutually exclusive with `css`.
+      #
+      # @param [Node, nil] within
+      #   The parent node to search within.
+      #
+      # @param [Float, Integer] timeout
+      #   How long to keep retrying, in seconds.
+      #
+      # @param [Float, Integer] interval
+      #   How long to sleep between attempts, in seconds.
+      #
+      # @return [Node, nil]
+      #   The matching node, or `nil` if `timeout` elapses without a match.
+      #
+      # @raise [ArgumentError]
+      #   If neither or both of `css`/`xpath` are given.
+      #
+      # @example
+      #   browser.go_to("https://github.com/")
+      #   browser.wait_for_selector(css: "a[aria-label='Issues you created']") # => Node
+      #
+      def wait_for_selector(css: nil, xpath: nil, within: nil, timeout: @page.timeout, interval: 0.05)
+        raise ArgumentError, "pass either `css:` or `xpath:`" if css && xpath
+
+        selector, method_name = css ? [css, :at_css] : [xpath, :at_xpath]
+        raise ArgumentError, "pass either `css:` or `xpath:`" unless selector
+
+        start = Utils::ElapsedTime.monotonic_time
+
+        until (node = send(method_name, selector, within: within))
+          return nil if Utils::ElapsedTime.timeout?(start, timeout)
+
+          sleep(interval)
+        end
+
+        node
+      end
+
       #
       # Adds a `<script>` tag to the document.
       #

data/lib/ferrum/frame/runtime.rb

@@ -3,15 +3,30 @@
 require "singleton"
 
 module Ferrum
+  #
+  # Placeholder object substituted for a JavaScript value that couldn't be
+  # fully serialized because it contains circular references. It exists
+  # only so that {#inspect} can report the situation instead of the
+  # evaluation raising or hanging.
+  #
   class CyclicObject
     include Singleton
 
+    # Debug representation of the singleton placeholder.
+    #
+    # @return [String]
     def inspect
       %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
     end
   end
 
   class Frame
+    #
+    # Evaluates and executes JavaScript in a frame's execution context via
+    # `Runtime.callFunctionOn`, converting arguments and return values
+    # between Ruby and JS, and resolving object/array/node results
+    # (including cyclic ones, via {CyclicObject}) into Ruby equivalents.
+    #
     module Runtime
       INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
       INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
@@ -86,10 +101,41 @@ module Ferrum
         true
       end
 
+      #
+      # Evaluates a raw JS function declaration (unlike {#evaluate}, which
+      # wraps the given expression in one), optionally on a specific remote
+      # object instead of the frame's global execution context.
+      #
+      # @param [String] expression
+      #   A JS function declaration, e.g. `"function(a, b) { return a + b }"`.
+      #
+      # @param [Array] args
+      #   Arguments to pass to the function.
+      #
+      # @param [Node, nil] on
+      #   Remote object to invoke the function on.
+      #
       def evaluate_func(expression, *args, on: nil)
         call(expression: expression, arguments: args, on: on)
       end
 
+      #
+      # Evaluates an expression against a given node's remote object (+this+
+      # refers to the node), returning the raw JS value rather than
+      # resolving it to a {Node}/Hash/Array.
+      #
+      # @param [Node] node
+      #   The node to evaluate the expression on.
+      #
+      # @param [String] expression
+      #   The JavaScript to evaluate.
+      #
+      # @param [Boolean] by_value
+      #   Whether to return the plain JS value instead of a handle.
+      #
+      # @param [Integer] wait
+      #   Passed through to the underlying `Runtime.callFunctionOn` command.
+      #
       def evaluate_on(node:, expression:, by_value: true, wait: 0)
         options = { handle: true }
         expression = format("function() { return %s }", expression)
@@ -100,7 +146,8 @@ module Ferrum
       private
 
       def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
-        errors = [NodeNotFoundError, NoExecutionContextError]
+        # do not rescue -> retry if we operate on an existing node
+        errors = on ? [] : [NodeNotFoundError, NoExecutionContextError]
 
         Utils::Attempt.with_retry(errors: errors, max: INTERMITTENT_ATTEMPTS, wait: INTERMITTENT_SLEEP) do
           params = options.dup
@@ -139,7 +186,7 @@ module Ferrum
         end
       end
 
-      def handle_response(response)
+      def handle_response(response, check_cyclic: true)
         case response["type"]
         when "boolean", "number", "string"
           response["value"]
@@ -152,18 +199,13 @@ module Ferrum
 
           case response["subtype"]
           when "node"
-            # We cannot store object_id in the node because page can be reloaded
-            # and node destroyed so we need to retrieve it each time for given id.
-            # Though we can try to subscribe to `DOM.childNodeRemoved` and
-            # `DOM.childNodeInserted` in the future.
-            node_id = @page.command("DOM.requestNode", objectId: object_id)["nodeId"]
-            description = @page.command("DOM.describeNode", nodeId: node_id)["node"]
-            Node.new(self, @page.target_id, node_id, description)
+            description = @page.command("DOM.describeNode", objectId: object_id)["node"]
+            Node.new(self, @page.target_id, description, object_id: object_id)
           when "array"
-            reduce_props(object_id, []) do |memo, key, value|
+            reduce_props(object_id, [], check_cyclic: check_cyclic) do |memo, key, value|
               next(memo) unless Integer(key, exception: false)
 
-              value = value["objectId"] ? handle_response(value) : value["value"]
+              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
               memo.insert(key.to_i, value)
             end.compact
           when "date"
@@ -171,8 +213,8 @@ module Ferrum
           when "null"
             nil
           else
-            reduce_props(object_id, {}) do |memo, key, value|
-              value = value["objectId"] ? handle_response(value) : value["value"]
+            reduce_props(object_id, {}, check_cyclic: check_cyclic) do |memo, key, value|
+              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
               memo.merge(key => value)
             end
           end
@@ -192,8 +234,8 @@ module Ferrum
         end
       end
 
-      def reduce_props(object_id, to)
-        if cyclic?(object_id).dig("result", "value")
+      def reduce_props(object_id, to, check_cyclic: true)
+        if check_cyclic && cyclic?(object_id).dig("result", "value")
           to.is_a?(Array) ? [cyclic_object] : cyclic_object
         else
           props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)

data/lib/ferrum/frame.rb

@@ -4,6 +4,12 @@ require "ferrum/frame/dom"
 require "ferrum/frame/runtime"
 
 module Ferrum
+  #
+  # Represents a frame (the main document or a nested `iframe`) within a
+  # {Page}. Each frame has its own execution context for JavaScript and its
+  # own lifecycle state, tracked via {#state}. DOM search and JS evaluation
+  # methods are provided by the included {DOM} and {Runtime} modules.
+  #
   class Frame
     include DOM
     include Runtime
@@ -12,6 +18,7 @@ module Ferrum
       started_loading
       navigated
       stopped_loading
+      canceled
     ].freeze
 
     # The Frame's unique id.
@@ -36,16 +43,36 @@ module Ferrum
 
     # One of the states frame's in.
     #
-    # @return [:started_loading, :navigated, :stopped_loading, nil]
+    # @return [:started_loading, :navigated, :stopped_loading, :canceled, nil]
     attr_reader :state
 
+    # Frame loader id.
+    #
+    # @return [String, nil]
+    attr_accessor :loader_id
+
+    # Frame's lifecycle events (navigation, load, paint, etc.).
+    #
+    # @return [Array<Hash{String => (String|Float)}>]
+    attr_reader :lifecycle_events
+
     def initialize(id, page, parent_id = nil)
       @id = id
       @page = page
       @parent_id = parent_id
+      @lifecycle_events = []
       @execution_id = Concurrent::MVar.new
     end
 
+    # Sets the frame's state, validating it's one of {STATE_VALUES}.
+    #
+    # @param [Symbol] value
+    #   One of `:started_loading`, `:navigated`, `:stopped_loading`,
+    #   `:canceled`.
+    #
+    # @return [Symbol]
+    #
+    # @raise [ArgumentError]
     def state=(value)
       raise ArgumentError unless STATE_VALUES.include?(value)
 
@@ -94,6 +121,20 @@ module Ferrum
       @parent_id.nil?
     end
 
+    #
+    # Returns whether the frame has finished loading (+:stopped_loading+ state).
+    # Frames in +:canceled+ state (execution context torn down mid-navigation)
+    # are not considered idle.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   browser.go_to("https://example.com")
+    #   browser.main_frame.idle? # => true
+    def idle?
+      state == :stopped_loading
+    end
+
     #
     # Returns the parent frame if this frame is nested in another one.
     #
@@ -164,6 +205,15 @@ module Ferrum
       value
     end
 
+    #
+    # Sets the execution context id, or clears it if +nil+ is given (e.g.
+    # when the context is torn down mid-navigation and hasn't been
+    # replaced yet).
+    #
+    # @param [Integer, nil] value
+    #
+    # @return [Integer, nil]
+    #
     def execution_id=(value)
       if value.nil?
         @execution_id.try_take!
@@ -172,11 +222,18 @@ module Ferrum
       end
     end
 
+    #
+    # Debug representation of the frame, including its internal state.
+    #
+    # @return [String]
+    #
     def inspect
       "#<#{self.class} " \
         "@id=#{@id.inspect} " \
         "@parent_id=#{@parent_id.inspect} " \
         "@name=#{@name.inspect} " \
+        "@loader_id=#{@loader_id.inspect} " \
+        "@lifecycle_events=#{@lifecycle_events.inspect} " \
         "@state=#{@state.inspect} " \
         "@execution_id=#{@execution_id.inspect}>"
     end

data/lib/ferrum/headers.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Manages the extra HTTP headers a page sends with its requests, via the
+  # CDP `Network.setExtraHTTPHeaders` command. Setting a `User-Agent` or
+  # `Accept-Language` header also updates the corresponding navigator
+  # override so JavaScript sees consistent values.
+  #
   class Headers
     def initialize(page)
       @page = page

data/lib/ferrum/interceptable.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ferrum
+  # Shared by {Page} and {Worker}: subscribes to the CDP events behind the
+  # `:request` (Fetch-domain request interception), `:response` (a request's
+  # response has fully loaded) and `:auth` (proxy/basic auth challenges)
+  # pseudo-events, on top of the includer's own `client` and `network`.
+  # Anything else is passed straight through to `client`.
+  module Interceptable
+    # Subscribes to a CDP event, or to `:request`/`:response`/`:auth`.
+    #
+    # @param [Symbol, String] name
+    #
+    # @yieldparam [Network::InterceptedRequest, Network::Exchange, Network::AuthRequest, Hash] arg
+    #   For `:request`, the intercepted {Network::InterceptedRequest}. For
+    #   `:response`, the finished {Network::Exchange}. For `:auth`, the
+    #   {Network::AuthRequest} challenge. For any other (raw CDP) event name,
+    #   the event's params `Hash`.
+    #
+    # @yieldparam [Integer] index
+    #   This callback's position among all callbacks currently registered for
+    #   the event.
+    #
+    # @yieldparam [Integer] total
+    #   Total number of callbacks currently registered for the event.
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    def on(name, &block)
+      case name
+      when :request
+        client.on("Fetch.requestPaused") do |params, index, total|
+          request = Network::InterceptedRequest.new(client, params)
+          exchange = network.find_or_build_exchange(request.network_id)
+          exchange.intercepted_request = request
+          block.call(request, index, total)
+        end
+      when :response
+        client.on("Network.loadingFinished") do |params, index, total|
+          exchange = network.select(params["requestId"]).last
+          block.call(exchange, index, total) if exchange
+        end
+      when :auth
+        client.on("Fetch.authRequired") do |params, index, total|
+          request = Network::AuthRequest.new(self, params)
+          block.call(request, index, total)
+        end
+      else
+        client.on(name, &block)
+      end
+    end
+
+    # Unsubscribes a listener previously registered via {#on}.
+    #
+    # @param [Symbol, String] name
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    def off(name, id)
+      case name
+      when :request
+        client.off("Fetch.requestPaused", id)
+      when :response
+        client.off("Network.loadingFinished", id)
+      when :auth
+        client.off("Fetch.authRequired", id)
+      else
+        client.off(name, id)
+      end
+    end
+
+    # Whether there's at least one callback registered for the event.
+    #
+    # @param [String] event
+    #
+    # @return [Boolean]
+    def subscribed?(event)
+      client.subscribed?(event)
+    end
+  end
+end

data/lib/ferrum/keyboard.rb

@@ -3,6 +3,12 @@
 require "json"
 
 module Ferrum
+  #
+  # Simulates keyboard input for a page via the CDP `Input.dispatchKeyEvent`
+  # command. Provides low-level `down`/`up` for individual keys as well as
+  # `type` for typing a sequence of characters/keys, translating key names
+  # and modifier chords into the CDP key event parameters Chrome expects.
+  #
   class Keyboard
     KEYS = JSON.parse(File.read(File.expand_path("keyboard.json", __dir__)))
     MODIFIERS = { "alt" => 1, "ctrl" => 2, "control" => 2,
@@ -136,6 +142,13 @@ module Ferrum
               modifiers: pressed.map { |k| MODIFIERS[k] }.reduce(0, :|)
             )
 
+            # Synthetic key events alone do not trigger the browser's editing
+            # shortcuts (e.g. select-all). When the chord matches a platform
+            # accelerator, name the editing command explicitly so Chrome
+            # executes it via the CDP `commands` field.
+            command = editing_command(pressed, char)
+            key = key.merge(commands: [command]) if command
+
             modifiers = pressed.map { |k| to_options(KEYS.fetch(KEYS_MAPPING[k.to_sym])) }
             modifiers + [to_options(key)]
           end.flatten
@@ -157,5 +170,17 @@ module Ferrum
     def to_options(hash)
       hash.inject({}) { |memo, (k, v)| memo.merge(k.to_sym => v) }
     end
+
+    # Names the CDP editing command for a modifier chord, or nil if the chord
+    # is not an editing accelerator. The accelerator modifier is Cmd on macOS
+    # and Ctrl elsewhere, matching Chrome's own key bindings. Currently only
+    # select-all is mapped; extend the guard for further commands as needed.
+    def editing_command(pressed, char)
+      return unless char.casecmp?("a")
+      return if pressed.include?("shift")
+
+      accelerator = Utils::Platform.mac? ? MODIFIERS["meta"] : MODIFIERS["ctrl"]
+      "selectAll" if pressed.map { |k| MODIFIERS[k] }.compact.uniq == [accelerator]
+    end
   end
 end