ferrum 0.17.2 → 0.18.0

data/lib/ferrum/frame.rb

@@ -4,6 +4,12 @@ require "ferrum/frame/dom"
 require "ferrum/frame/runtime"
 
 module Ferrum
+  #
+  # Represents a frame (the main document or a nested `iframe`) within a
+  # {Page}. Each frame has its own execution context for JavaScript and its
+  # own lifecycle state, tracked via {#state}. DOM search and JS evaluation
+  # methods are provided by the included {DOM} and {Runtime} modules.
+  #
   class Frame
     include DOM
     include Runtime
@@ -12,6 +18,7 @@ module Ferrum
       started_loading
       navigated
       stopped_loading
+      canceled
     ].freeze
 
     # The Frame's unique id.
@@ -36,16 +43,36 @@ module Ferrum
 
     # One of the states frame's in.
     #
-    # @return [:started_loading, :navigated, :stopped_loading, nil]
+    # @return [:started_loading, :navigated, :stopped_loading, :canceled, nil]
     attr_reader :state
 
+    # Frame loader id.
+    #
+    # @return [String, nil]
+    attr_accessor :loader_id
+
+    # Frame's lifecycle events (navigation, load, paint, etc.).
+    #
+    # @return [Array<Hash{String => (String|Float)}>]
+    attr_reader :lifecycle_events
+
     def initialize(id, page, parent_id = nil)
       @id = id
       @page = page
       @parent_id = parent_id
+      @lifecycle_events = []
       @execution_id = Concurrent::MVar.new
     end
 
+    # Sets the frame's state, validating it's one of {STATE_VALUES}.
+    #
+    # @param [Symbol] value
+    #   One of `:started_loading`, `:navigated`, `:stopped_loading`,
+    #   `:canceled`.
+    #
+    # @return [Symbol]
+    #
+    # @raise [ArgumentError]
     def state=(value)
       raise ArgumentError unless STATE_VALUES.include?(value)
 
@@ -94,6 +121,20 @@ module Ferrum
       @parent_id.nil?
     end
 
+    #
+    # Returns whether the frame has finished loading (+:stopped_loading+ state).
+    # Frames in +:canceled+ state (execution context torn down mid-navigation)
+    # are not considered idle.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   browser.go_to("https://example.com")
+    #   browser.main_frame.idle? # => true
+    def idle?
+      state == :stopped_loading
+    end
+
     #
     # Returns the parent frame if this frame is nested in another one.
     #
@@ -164,6 +205,15 @@ module Ferrum
       value
     end
 
+    #
+    # Sets the execution context id, or clears it if +nil+ is given (e.g.
+    # when the context is torn down mid-navigation and hasn't been
+    # replaced yet).
+    #
+    # @param [Integer, nil] value
+    #
+    # @return [Integer, nil]
+    #
     def execution_id=(value)
       if value.nil?
         @execution_id.try_take!
@@ -172,11 +222,18 @@ module Ferrum
       end
     end
 
+    #
+    # Debug representation of the frame, including its internal state.
+    #
+    # @return [String]
+    #
     def inspect
       "#<#{self.class} " \
         "@id=#{@id.inspect} " \
         "@parent_id=#{@parent_id.inspect} " \
         "@name=#{@name.inspect} " \
+        "@loader_id=#{@loader_id.inspect} " \
+        "@lifecycle_events=#{@lifecycle_events.inspect} " \
         "@state=#{@state.inspect} " \
         "@execution_id=#{@execution_id.inspect}>"
     end

data/lib/ferrum/headers.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Manages the extra HTTP headers a page sends with its requests, via the
+  # CDP `Network.setExtraHTTPHeaders` command. Setting a `User-Agent` or
+  # `Accept-Language` header also updates the corresponding navigator
+  # override so JavaScript sees consistent values.
+  #
   class Headers
     def initialize(page)
       @page = page

data/lib/ferrum/interceptable.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Ferrum
+  # Shared by {Page} and {Worker}: subscribes to the CDP events behind the
+  # `:request` (Fetch-domain request interception) and `:auth` (proxy/basic
+  # auth challenges) pseudo-events, on top of the includer's own `client`
+  # and `network`. Anything else is passed straight through to `client`.
+  module Interceptable
+    # Subscribes to a CDP event, or to `:request`/`:auth`.
+    #
+    # @param [Symbol, String] name
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    def on(name, &block)
+      case name
+      when :request
+        client.on("Fetch.requestPaused") do |params, index, total|
+          request = Network::InterceptedRequest.new(client, params)
+          exchange = network.find_or_build_exchange(request.network_id)
+          exchange.intercepted_request = request
+          block.call(request, index, total)
+        end
+      when :auth
+        client.on("Fetch.authRequired") do |params, index, total|
+          request = Network::AuthRequest.new(self, params)
+          block.call(request, index, total)
+        end
+      else
+        client.on(name, &block)
+      end
+    end
+
+    # Unsubscribes a listener previously registered via {#on}.
+    #
+    # @param [Symbol, String] name
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    def off(name, id)
+      case name
+      when :request
+        client.off("Fetch.requestPaused", id)
+      when :auth
+        client.off("Fetch.authRequired", id)
+      else
+        client.off(name, id)
+      end
+    end
+
+    # Whether there's at least one callback registered for the event.
+    #
+    # @param [String] event
+    #
+    # @return [Boolean]
+    def subscribed?(event)
+      client.subscribed?(event)
+    end
+  end
+end

data/lib/ferrum/keyboard.rb

@@ -3,6 +3,12 @@
 require "json"
 
 module Ferrum
+  #
+  # Simulates keyboard input for a page via the CDP `Input.dispatchKeyEvent`
+  # command. Provides low-level `down`/`up` for individual keys as well as
+  # `type` for typing a sequence of characters/keys, translating key names
+  # and modifier chords into the CDP key event parameters Chrome expects.
+  #
   class Keyboard
     KEYS = JSON.parse(File.read(File.expand_path("keyboard.json", __dir__)))
     MODIFIERS = { "alt" => 1, "ctrl" => 2, "control" => 2,
@@ -136,6 +142,13 @@ module Ferrum
               modifiers: pressed.map { |k| MODIFIERS[k] }.reduce(0, :|)
             )
 
+            # Synthetic key events alone do not trigger the browser's editing
+            # shortcuts (e.g. select-all). When the chord matches a platform
+            # accelerator, name the editing command explicitly so Chrome
+            # executes it via the CDP `commands` field.
+            command = editing_command(pressed, char)
+            key = key.merge(commands: [command]) if command
+
             modifiers = pressed.map { |k| to_options(KEYS.fetch(KEYS_MAPPING[k.to_sym])) }
             modifiers + [to_options(key)]
           end.flatten
@@ -157,5 +170,17 @@ module Ferrum
     def to_options(hash)
       hash.inject({}) { |memo, (k, v)| memo.merge(k.to_sym => v) }
     end
+
+    # Names the CDP editing command for a modifier chord, or nil if the chord
+    # is not an editing accelerator. The accelerator modifier is Cmd on macOS
+    # and Ctrl elsewhere, matching Chrome's own key bindings. Currently only
+    # select-all is mapped; extend the guard for further commands as needed.
+    def editing_command(pressed, char)
+      return unless char.casecmp?("a")
+      return if pressed.include?("shift")
+
+      accelerator = Utils::Platform.mac? ? MODIFIERS["meta"] : MODIFIERS["ctrl"]
+      "selectAll" if pressed.map { |k| MODIFIERS[k] }.compact.uniq == [accelerator]
+    end
   end
 end

data/lib/ferrum/mouse.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Simulates mouse input for a page via the CDP `Input.dispatchMouseEvent`
+  # command, and page scrolling via JavaScript. Keeps track of the current
+  # pointer position and pressed buttons so `move`, `down`, `up` and `click`
+  # can be composed while reporting consistent state to Chrome.
+  #
   class Mouse
     CLICK_WAIT = ENV.fetch("FERRUM_CLICK_WAIT", 0.1).to_f
     BUTTON_MASKS = {

data/lib/ferrum/network/auth_request.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Network
+    #
+    # Represents an HTTP authentication challenge (basic or digest auth
+    # prompt) raised by the browser via `Fetch.authRequired`, which can be
+    # answered with credentials or canceled.
+    #
     class AuthRequest
       attr_accessor :request_id, :frame_id, :resource_type
 
@@ -14,47 +19,121 @@ module Ferrum
         @request = params["request"]
       end
 
+      #
+      # Whether this request is for the navigation of a frame, as opposed
+      # to a subresource (script, image, XHR, etc).
+      #
+      # @return [Boolean]
+      #
       def navigation_request?
         @params["isNavigationRequest"]
       end
 
+      #
+      # Whether the authentication challenge's source matches the given
+      # type.
+      #
+      # @param [String, Symbol] source
+      #   One of `:server` or `:proxy`.
+      #
+      # @return [Boolean]
+      #
       def auth_challenge?(source)
         @params.dig("authChallenge", "source")&.downcase&.to_s == source.to_s
       end
 
-      def match?(regexp)
-        !!url.match(regexp)
+      #
+      # Whether the request's URL matches the given pattern.
+      #
+      # @param [String, Regexp] pattern
+      #
+      # @return [Boolean]
+      #
+      def match?(pattern)
+        case url
+        when pattern
+          true
+        else
+          false
+        end
       end
 
+      #
+      # Responds to the authentication challenge, e.g. with credentials or by
+      # canceling it.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :response
+      #   One of `"Default"`, `"CancelAuth"`, or `"ProvideCredentials"`.
+      #
+      # @option options [String] :username
+      #
+      # @option options [String] :password
+      #
       def continue(**options)
         options = options.merge(requestId: request_id)
         @page.command("Fetch.continueWithAuth", **options)
       end
 
+      #
+      # Cancels the authentication challenge and the request.
+      #
       def abort
         @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
       end
 
+      #
+      # The URL for the request.
+      #
+      # @return [String]
+      #
       def url
         @request["url"]
       end
 
+      #
+      # The request method.
+      #
+      # @return [String]
+      #
       def method
         @request["method"]
       end
 
+      #
+      # The request headers.
+      #
+      # @return [Hash{String => String}]
+      #
       def headers
         @request["headers"]
       end
 
+      #
+      # The request's initial priority, one of `"VeryLow"`, `"Low"`,
+      # `"Medium"`, `"High"`, or `"VeryHigh"`.
+      #
+      # @return [String]
+      #
       def initial_priority
         @request["initialPriority"]
       end
 
+      #
+      # The request's referrer policy.
+      #
+      # @return [String]
+      #
       def referrer_policy
         @request["referrerPolicy"]
       end
 
+      #
+      # Inspects the auth request.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@request_id=#{@request_id.inspect} " \

data/lib/ferrum/network/error.rb

@@ -2,15 +2,30 @@
 
 module Ferrum
   class Network
+    #
+    # Represents a network-level loading error reported for a request,
+    # e.g. via `Network.loadingFailed`. Distinct from the Ruby exceptions
+    # defined in `ferrum/errors.rb`.
+    #
     class Error
       attr_writer :canceled
       attr_reader :time, :timestamp
       attr_accessor :id, :url, :type, :error_text, :monotonic_time, :description
 
+      # Whether the request was canceled.
+      #
+      # @return [Boolean]
       def canceled?
         @canceled
       end
 
+      # Sets the error's timestamp, deriving {#time} from it.
+      #
+      # @param [Float] value
+      #   Timestamp in milliseconds since epoch, as reported by
+      #   `Log.entryAdded`.
+      #
+      # @return [Float]
       def timestamp=(value)
         @timestamp = value
         @time = Time.strptime((value / 1000).to_s, "%s")

data/lib/ferrum/network/exchange.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Network
+    #
+    # Pairs a {Request} with its eventual {Response} or {Error} (and, if
+    # interception is enabled, its {InterceptedRequest}), representing the
+    # full lifecycle of a single network request.
+    #
     class Exchange
       # ID of the request.
       #
@@ -34,6 +39,11 @@ module Ferrum
       # @return Boolean
       attr_accessor :unknown
 
+      # The raw `Network.requestWillBeSentExtraInfo` params for the request,
+      # if it arrived before `request` was set.
+      #
+      # @return [Hash, nil]
+      #
       # @api private
       attr_accessor :request_extra_info
 

data/lib/ferrum/network/intercepted_request.rb

@@ -5,6 +5,11 @@ require "base64"
 
 module Ferrum
   class Network
+    #
+    # Represents a request paused by request interception (`Fetch.enable`),
+    # allowing it to be continued as is or with overrides, fulfilled with a
+    # fake response, or aborted before it reaches the network.
+    #
     class InterceptedRequest
       include RequestParams
 
@@ -21,18 +26,62 @@ module Ferrum
         @network_id = params["networkId"]
       end
 
+      #
+      # Whether the request's current status matches the given value.
+      #
+      # @param [String, Symbol] value
+      #   One of `:responded`, `:continued`, `:aborted`.
+      #
+      # @return [Boolean]
+      #
       def status?(value)
         @status == value.to_sym
       end
 
+      #
+      # Whether this request is for the navigation of a frame, as opposed
+      # to a subresource (script, image, XHR, etc).
+      #
+      # @return [Boolean]
+      #
       def navigation_request?
         @params["isNavigationRequest"]
       end
 
-      def match?(regexp)
-        !!url.match(regexp)
+      #
+      # Whether the request's URL matches the given pattern.
+      #
+      # @param [String, Regexp] pattern
+      #
+      # @return [Boolean]
+      #
+      def match?(pattern)
+        case url
+        when pattern
+          true
+        else
+          false
+        end
       end
 
+      #
+      # Fulfills the intercepted request with a fake response instead of
+      # letting it reach the network.
+      #
+      # @param [Hash] options
+      #
+      # @option options [Integer] :responseCode
+      #   HTTP status code to respond with, `200` by default.
+      #
+      # @option options [String] :body
+      #   Response body.
+      #
+      # @option options [Hash] :responseHeaders
+      #   Response headers.
+      #
+      # @example
+      #   request.respond(body: "Lorem ipsum")
+      #
       def respond(**options)
         has_body = options.key?(:body)
         headers = has_body ? { "content-length" => options.fetch(:body, "").length } : {}
@@ -46,25 +95,68 @@ module Ferrum
         @client.command("Fetch.fulfillRequest", async: true, **options)
       end
 
+      #
+      # Continues the intercepted request, letting it reach the network as
+      # is, or with the given overrides.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :url
+      #   Overrides the request's URL.
+      #
+      # @option options [String] :method
+      #   Overrides the request's method.
+      #
+      # @option options [String] :postData
+      #   Overrides the request's post data.
+      #
+      # @option options [Hash] :headers
+      #   Overrides the request's headers.
+      #
+      # @example
+      #   request.continue
+      #
       def continue(**options)
         options = options.merge(requestId: request_id)
         @status = :continued
         @client.command("Fetch.continueRequest", async: true, **options)
       end
 
+      #
+      # Aborts the intercepted request.
+      #
+      # @example
+      #   request.abort
+      #
       def abort
         @status = :aborted
         @client.command("Fetch.failRequest", async: true, requestId: request_id, errorReason: "BlockedByClient")
       end
 
+      #
+      # The request's initial priority, one of `"VeryLow"`, `"Low"`,
+      # `"Medium"`, `"High"`, or `"VeryHigh"`.
+      #
+      # @return [String]
+      #
       def initial_priority
         @request["initialPriority"]
       end
 
+      #
+      # The request's referrer policy.
+      #
+      # @return [String]
+      #
       def referrer_policy
         @request["referrerPolicy"]
       end
 
+      #
+      # Inspects the intercepted request.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@request_id=#{@request_id.inspect} " \

data/lib/ferrum/network/request.rb

@@ -33,7 +33,7 @@ module Ferrum
       end
 
       #
-      # The request resouce type.
+      # The request resource type.
       #
       # @return [String]
       #

data/lib/ferrum/network/response.rb

@@ -145,6 +145,8 @@ module Ferrum
         # nop
       end
 
+      #
+      # Whether this response is the main frame's current response.
       #
       # @return [Boolean]
       #