ferrum 0.17.2 → 0.18.0

data/lib/ferrum/contexts.rb

@@ -3,41 +3,79 @@
 require "ferrum/context"
 
 module Ferrum
+  #
+  # Owns the browser's whole collection of {Context}s, keyed by their
+  # `browserContextId`. Subscribes to the CDP `Target.*` events, creates
+  # {Context}s and {Target}s as they're discovered/attached, and routes
+  # target lifecycle updates (info changed, destroyed, crashed) to the
+  # {Context} they belong to.
+  #
   class Contexts
-    ALLOWED_TARGET_TYPES = %w[page iframe].freeze
+    ALLOWED_TARGET_TYPES = %w[page iframe worker shared_worker service_worker].freeze
+    RECURSIVE_AUTO_ATTACH_TYPES = %w[page iframe worker shared_worker].freeze
 
     include Enumerable
 
     attr_reader :contexts
 
     def initialize(client)
-      @contexts = Concurrent::Map.new
       @client = client
+      @contexts = Concurrent::Map.new
+      @manually_attached = Concurrent::Map.new
       subscribe
       auto_attach
       discover
     end
 
+    # Marks a target, so the next time we see it attached, we leave its session alone instead of {#detach}ing it.
+    # Used by {Context#attach_target} right before it manually attaches to a service worker on the caller's behalf.
+    def manually_attached(target_id)
+      @manually_attached[target_id] = true
+    end
+
+    # The browser's first context, created lazily.
+    #
+    # @return [Context]
     def default_context
       @default_context ||= create
     end
 
+    # Iterates over `[id, context]` pairs; returns an `Enumerator` if no
+    # block is given.
+    #
+    # @return [void, Enumerator]
     def each(&)
       return enum_for(__method__) unless block_given?
 
       @contexts.each(&)
     end
 
+    # The context with the given id, if any.
+    #
+    # @param [String] id
+    #
+    # @return [Context, nil]
     def [](id)
       @contexts[id]
     end
 
+    # The context that owns the given target, if any.
+    #
+    # @param [String] target_id
+    #
+    # @return [Context, nil]
     def find_by(target_id:)
       context = nil
       @contexts.each_value { |c| context = c if c.target?(target_id) }
       context
     end
 
+    # Creates a new browser context (like an incognito profile).
+    #
+    # @param [Hash] options
+    #   Keyword arguments forwarded to `Target.createBrowserContext`.
+    #
+    # @return [Context]
     def create(**options)
       response = @client.command("Target.createBrowserContext", **options)
       context_id = response["browserContextId"]
@@ -46,6 +84,11 @@ module Ferrum
       context
     end
 
+    # Disposes a browser context and all of its targets.
+    #
+    # @param [String] context_id
+    #
+    # @return [Boolean]
     def dispose(context_id)
       context = @contexts[context_id]
       return unless context
@@ -56,36 +99,54 @@ module Ferrum
       true
     end
 
+    # Closes the WebSocket connection of every target in every context,
+    # without disposing the contexts themselves.
+    #
+    # @return [void]
     def close_connections
       @contexts.each_value(&:close_targets_connection)
     end
 
+    # Disposes every context still known to the browser.
+    #
+    # @return [void]
     def reset
       context_ids = @client.command("Target.getBrowserContexts")["browserContextIds"]
       @default_context = nil if context_ids.include?(@default_context&.id)
       @contexts.each_key { |id| dispose(id) if context_ids.include?(id) }
     end
 
+    # Number of known contexts.
+    #
+    # @return [Integer]
     def size
       @contexts.size
     end
 
     private
 
-    def subscribe # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
-      @client.on("Target.attachedToTarget") do |params|
+    def subscribe
+      subscribe_attached_target(@client)
+      subscribe_target_created
+    end
+
+    # Registered once on the top-level client, and again on every page's/
+    # worker's own session once we re-arm auto-attach on it.
+    def subscribe_attached_target(client)
+      client.on("Target.attachedToTarget") do |params|
         info, session_id = params.values_at("targetInfo", "sessionId")
         next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
         add_context(context_id)
+        target = @contexts[context_id]&.add_target(session_id: session_id, params: info)
 
-        @contexts[context_id]&.add_target(session_id: session_id, params: info)
-        if params["waitingForDebugger"]
-          @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
-        end
+        rearm_auto_attach(session_id, info["type"])
+        handle_attach(target, session_id, params)
       end
+    end
 
+    def subscribe_target_created
       @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
         next unless ALLOWED_TARGET_TYPES.include?(info["type"])
@@ -95,7 +156,7 @@ module Ferrum
 
         if info["type"] == "iframe" &&
            (target = @contexts[context_id]&.find_target { |t| t.connected? && t.page.frame_by(id: info["targetId"]) })
-          @contexts[context_id]&.add_target(session_id: target.page.client.session_id, params: info)
+          @contexts[context_id]&.add_target(session_id: target.session_id, params: info)
         else
           @contexts[context_id]&.add_target(params: info)
         end
@@ -120,6 +181,55 @@ module Ferrum
       end
     end
 
+    def rearm_auto_attach(session_id, type)
+      return unless RECURSIVE_AUTO_ATTACH_TYPES.include?(type)
+
+      client = @client.session(session_id)
+      client.command("Target.setAutoAttach", autoAttach: true, waitForDebuggerOnStart: true, flatten: true, async: true)
+      subscribe_attached_target(client)
+    end
+
+    def handle_attach(target, session_id, params)
+      return unless target
+
+      if target.service_worker?
+        detach_unless_manually_attached(target, session_id)
+      elsif target.worker? || target.shared_worker?
+        connect_worker(target)
+      elsif params["waitingForDebugger"]
+        resume(session_id)
+      end
+    end
+
+    # Attaching keeps a service worker alive forever, so unless the caller
+    # explicitly asked to connect to it (via Context#attach_target), we
+    # just resume it and let go.
+    def detach_unless_manually_attached(target, session_id)
+      return if @manually_attached.delete(target.id)
+
+      detach(session_id)
+    rescue BrowserError
+      nil
+    end
+
+    # Workers have no events to notify us when they're ready, so we
+    # connect right away. Worker#prepare enables the Network domain and
+    # only then resumes the debugger itself.
+    def connect_worker(target)
+      target.worker
+    rescue BrowserError
+      nil
+    end
+
+    def resume(session_id)
+      @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
+    end
+
+    def detach(session_id)
+      resume(session_id)
+      @client.command("Target.detachFromTarget", sessionId: session_id)
+    end
+
     def discover
       @client.command("Target.setDiscoverTargets", discover: true)
     end

data/lib/ferrum/cookies/cookie.rb

@@ -122,6 +122,8 @@ module Ferrum
         @attributes["priority"]
       end
 
+      #
+      # Specifies whether the cookie is a first-party set cookie or not.
       #
       # @return [Boolean]
       #
@@ -131,6 +133,8 @@ module Ferrum
 
       alias same_party? sameparty?
 
+      #
+      # The scheme (`"Unset"`, `"NonSecure"`, `"Secure"`) that the cookie was set with.
       #
       # @return [String]
       #
@@ -138,6 +142,8 @@ module Ferrum
         @attributes["sourceScheme"]
       end
 
+      #
+      # The port that the cookie was set with, or `-1` if unknown.
       #
       # @return [Integer]
       #

data/lib/ferrum/cookies.rb

@@ -4,6 +4,11 @@ require "yaml"
 require "ferrum/cookies/cookie"
 
 module Ferrum
+  #
+  # Manages a page's cookies via the CDP `Network` domain. Enumerable over
+  # the page's current {Cookies::Cookie}s, and provides methods to read,
+  # set, remove and clear them, as well as persist them to/from a YAML file.
+  #
   class Cookies
     include Enumerable
 

data/lib/ferrum/dialog.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents a JavaScript dialog (`alert`, `confirm`, `prompt` or
+  # `beforeunload`) shown by the page. Instances are yielded to blocks
+  # registered with `page.on(:dialog)`, and can be accepted (optionally with
+  # a prompt's text) or dismissed.
+  #
   class Dialog
     attr_reader :message, :default_prompt
 
@@ -51,8 +57,18 @@ module Ferrum
       @page.command("Page.handleJavaScriptDialog", slowmoable: true, accept: false)
     end
 
-    def match?(regexp)
-      !!message.match(regexp)
+    #
+    # Whether the dialog's message matches the given pattern.
+    #
+    # @param [Regexp, String] pattern
+    #   A regexp matched against the message, or a string checked for inclusion in it.
+    #
+    # @return [Boolean]
+    #
+    def match?(pattern)
+      return message.match?(pattern) if pattern.is_a?(Regexp)
+
+      message.include?(pattern.to_s)
     end
   end
 end

data/lib/ferrum/downloads.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Tracks files downloaded by a page. Configures the browser's download
+  # behavior/destination via {#set_behavior}, subscribes to the
+  # `Browser.downloadWillBegin`/`Browser.downloadProgress` CDP events to
+  # record their progress in {#files}, and lets callers block until the
+  # current download finishes via {#wait}.
+  #
   class Downloads
     VALID_BEHAVIOR = %i[deny allow allowAndName default].freeze
 
@@ -10,10 +17,26 @@ module Ferrum
       @files = {}
     end
 
+    #
+    # Returns information about all downloaded files.
+    #
+    # @return [Array<Hash>]
+    #
     def files
       @files.values
     end
 
+    #
+    # Waits until the current download finishes.
+    #
+    # @param [Integer] timeout
+    #   How long to wait in seconds.
+    #
+    # @yield
+    #   Optional block that triggers the download, e.g. clicking a link.
+    #
+    # @return [void]
+    #
     def wait(timeout = 5)
       @event.reset
       yield if block_given?
@@ -21,6 +44,20 @@ module Ferrum
       @event.set
     end
 
+    #
+    # Sets the browser's download behavior and destination directory.
+    #
+    # @param [String] save_path
+    #   Absolute path to the directory downloads should be saved to.
+    #
+    # @param [:deny, :allow, :allowAndName, :default] behavior
+    #   Whether/how to allow downloads.
+    #
+    # @return [void]
+    #
+    # @raise [ArgumentError]
+    # @raise [Ferrum::Error]
+    #
     def set_behavior(save_path:, behavior: :allow)
       raise ArgumentError unless VALID_BEHAVIOR.include?(behavior.to_sym)
       raise Error, "supply absolute path for `:save_path` option" unless Pathname.new(save_path.to_s).absolute?
@@ -32,11 +69,21 @@ module Ferrum
                     eventsEnabled: true)
     end
 
+    #
+    # Subscribes to download related CDP events.
+    #
+    # @return [void]
+    #
     def subscribe
       subscribe_download_will_begin
       subscribe_download_progress
     end
 
+    #
+    # Subscribes to the `Browser.downloadWillBegin` event to track new downloads.
+    #
+    # @return [void]
+    #
     def subscribe_download_will_begin
       @page.on("Browser.downloadWillBegin") do |params|
         @event.reset
@@ -44,6 +91,11 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to the `Browser.downloadProgress` event to track download state.
+    #
+    # @return [void]
+    #
     def subscribe_download_progress
       @page.on("Browser.downloadProgress") do |params|
         @files[params["guid"]].merge!(params)

data/lib/ferrum/errors.rb

@@ -1,47 +1,73 @@
 # frozen_string_literal: true
 
 module Ferrum
+  # Base class for all errors raised by Ferrum.
   class Error               < StandardError; end
+  # Raised when the referenced page no longer exists.
   class NoSuchPageError     < Error; end
+  # Raised when the referenced browser target cannot be found.
   class NoSuchTargetError   < Error; end
+  # Raised when a requested feature isn't implemented for the current setup.
   class NotImplementedError < Error; end
+  # Raised when no browser binary can be found at the configured path.
   class BinaryNotFoundError < Error; end
+  # Raised when a required file path option is empty.
   class EmptyPathError      < Error; end
+  # Raised when the browser process server fails to start or crashes.
   class ServerError         < Error; end
 
+  # Raised when a request fails to reach the server, e.g. due to DNS or
+  # connectivity issues.
   class StatusError < Error
     def initialize(url, message = nil)
       super(message || "Request to #{url} failed to reach server, check DNS and server status")
     end
   end
 
+  # Raised when the request reached the server, but the page still has
+  # pending network connections that never settled.
   class PendingConnectionsError < StatusError
     attr_reader :pendings
 
     def initialize(url, pendings = [])
       @pendings = pendings
-
-      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+      message = "Request to #{url} reached server, but there are still pending connections"
+      message += ": #{pendings.join(', ')}" unless @pendings.empty?
 
       super(url, message)
     end
   end
 
+  # Raised when waiting for a response from the browser times out.
   class TimeoutError < Error
+    #
+    # Explains that waiting for a response timed out.
+    #
+    # @return [String]
+    #
     def message
       "Timed out waiting for response. It's possible that this happened " \
         "because something took a very long time (for example a page load " \
         "was slow). If so, setting the :timeout option to a higher value might " \
-        "help."
+        "help. If this happened on an internal protocol call instead, try " \
+        "raising :protocol_timeout."
     end
   end
 
+  # Raised when an evaluated script takes too long to return a value.
   class ScriptTimeoutError < Error
+    #
+    # Explains that the evaluated script timed out.
+    #
+    # @return [String]
+    #
     def message
       "Timed out waiting for evaluated script to return a value"
     end
   end
 
+  # Raised when the browser process doesn't produce a websocket URL within
+  # the configured `:process_timeout`.
   class ProcessTimeoutError < Error
     attr_reader :output
 
@@ -51,12 +77,16 @@ module Ferrum
     end
   end
 
+  # Raised when trying to interact with a browser process that has died or
+  # a window that has already been closed.
   class DeadBrowserError < Error
     def initialize(message = "Browser is dead or given window is closed")
       super
     end
   end
 
+  # Raised when a node keeps moving between attempts to interact with it,
+  # e.g. while trying to click it.
   class NodeMovingError < Error
     def initialize(node, prev, current)
       @node = node
@@ -65,6 +95,11 @@ module Ferrum
       super(message)
     end
 
+    #
+    # Explains that the node moved between attempts to interact with it.
+    #
+    # @return [String]
+    #
     def message
       "#{@node.inspect} that you're trying to click is moving, hence " \
         "we cannot. Previously it was at #{@prev.inspect} but now at " \
@@ -72,12 +107,15 @@ module Ferrum
     end
   end
 
+  # Raised when the content quads (coordinates) for a node cannot be computed.
   class CoordinatesNotFoundError < Error
     def initialize(message = "Could not compute content quads")
       super
     end
   end
 
+  # Raised when the `:format` option passed to a screenshot call is not one
+  # of the supported formats.
   class InvalidScreenshotFormatError < Error
     def initialize(format)
       valid_formats = Page::Screenshot::SUPPORTED_SCREENSHOT_FORMAT.join(" | ")
@@ -85,6 +123,8 @@ module Ferrum
     end
   end
 
+  # Raised when the browser returns an error response over CDP. Wraps the
+  # raw response so callers can inspect its code and data.
   class BrowserError < Error
     attr_reader :response
 
@@ -93,23 +133,38 @@ module Ferrum
       super(response["message"])
     end
 
+    #
+    # Error code from the raw CDP error response.
+    #
+    # @return [Integer, nil]
+    #
     def code
       response["code"]
     end
 
+    #
+    # Additional data from the raw CDP error response.
+    #
+    # @return [Object, nil]
+    #
     def data
       response["data"]
     end
   end
 
+  # Raised when the browser reports that a DOM node no longer exists.
   class NodeNotFoundError < BrowserError; end
 
+  # Raised when there's no JavaScript execution context available to run a
+  # command against, e.g. because the frame isn't attached.
   class NoExecutionContextError < BrowserError
     def initialize(response = nil)
       super(response || { "message" => "There's no context available" })
     end
   end
 
+  # Raised when JavaScript evaluated in the page throws an exception. Carries
+  # the JS exception's class name, message, and stack trace.
   class JavaScriptError < BrowserError
     attr_reader :class_name, :message, :stack_trace
 

data/lib/ferrum/frame/dom.rb

@@ -19,6 +19,12 @@
 # details (DOM.describeNode).
 module Ferrum
   class Frame
+    #
+    # Evaluates and executes JavaScript to query and manipulate a frame's
+    # DOM: reading the URL, title, doctype, and HTML, finding nodes by
+    # XPath or CSS selector, and injecting `<script>`, `<style>`, and
+    # `<link>` tags.
+    #
     module DOM
       SCRIPT_SRC_TAG = <<~JS
         const script = document.createElement("script");
@@ -77,6 +83,17 @@ module Ferrum
         evaluate("window.top.document.title")
       end
 
+      #
+      # Returns current document's doctype declaration.
+      #
+      # @return [String, nil]
+      #   The serialized `<!DOCTYPE ...>` declaration, or +nil+ if the
+      #   document has none.
+      #
+      # @example
+      #   browser.go_to("https://example.com")
+      #   browser.doctype # => "<!DOCTYPE html>"
+      #
       def doctype
         evaluate("document.doctype && new XMLSerializer().serializeToString(document.doctype)")
       end

data/lib/ferrum/frame/runtime.rb

@@ -3,15 +3,30 @@
 require "singleton"
 
 module Ferrum
+  #
+  # Placeholder object substituted for a JavaScript value that couldn't be
+  # fully serialized because it contains circular references. It exists
+  # only so that {#inspect} can report the situation instead of the
+  # evaluation raising or hanging.
+  #
   class CyclicObject
     include Singleton
 
+    # Debug representation of the singleton placeholder.
+    #
+    # @return [String]
     def inspect
       %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
     end
   end
 
   class Frame
+    #
+    # Evaluates and executes JavaScript in a frame's execution context via
+    # `Runtime.callFunctionOn`, converting arguments and return values
+    # between Ruby and JS, and resolving object/array/node results
+    # (including cyclic ones, via {CyclicObject}) into Ruby equivalents.
+    #
     module Runtime
       INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
       INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
@@ -86,10 +101,41 @@ module Ferrum
         true
       end
 
+      #
+      # Evaluates a raw JS function declaration (unlike {#evaluate}, which
+      # wraps the given expression in one), optionally on a specific remote
+      # object instead of the frame's global execution context.
+      #
+      # @param [String] expression
+      #   A JS function declaration, e.g. `"function(a, b) { return a + b }"`.
+      #
+      # @param [Array] args
+      #   Arguments to pass to the function.
+      #
+      # @param [Node, nil] on
+      #   Remote object to invoke the function on.
+      #
       def evaluate_func(expression, *args, on: nil)
         call(expression: expression, arguments: args, on: on)
       end
 
+      #
+      # Evaluates an expression against a given node's remote object (+this+
+      # refers to the node), returning the raw JS value rather than
+      # resolving it to a {Node}/Hash/Array.
+      #
+      # @param [Node] node
+      #   The node to evaluate the expression on.
+      #
+      # @param [String] expression
+      #   The JavaScript to evaluate.
+      #
+      # @param [Boolean] by_value
+      #   Whether to return the plain JS value instead of a handle.
+      #
+      # @param [Integer] wait
+      #   Passed through to the underlying `Runtime.callFunctionOn` command.
+      #
       def evaluate_on(node:, expression:, by_value: true, wait: 0)
         options = { handle: true }
         expression = format("function() { return %s }", expression)
@@ -152,13 +198,8 @@ module Ferrum
 
           case response["subtype"]
           when "node"
-            # We cannot store object_id in the node because page can be reloaded
-            # and node destroyed so we need to retrieve it each time for given id.
-            # Though we can try to subscribe to `DOM.childNodeRemoved` and
-            # `DOM.childNodeInserted` in the future.
-            node_id = @page.command("DOM.requestNode", objectId: object_id)["nodeId"]
-            description = @page.command("DOM.describeNode", nodeId: node_id)["node"]
-            Node.new(self, @page.target_id, node_id, description)
+            description = @page.command("DOM.describeNode", objectId: object_id)["node"]
+            Node.new(self, @page.target_id, description, object_id: object_id)
           when "array"
             reduce_props(object_id, []) do |memo, key, value|
               next(memo) unless Integer(key, exception: false)