ferrum 0.17.2 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbed705a172379d21bd97f46bdd7ed29e444b08d79e6191ecdaf1e3cdc254793
-  data.tar.gz: d5143b49451e8c7a31ae394ac226a8c2fb8feda6c7b30aee35e7349e3c5c818f
+  metadata.gz: 4a719b34dd1893e7c0ba2b71e121e1a2b7cc7950b96a1e462ac6a3a9f36c04e0
+  data.tar.gz: 4c96a549ff3aa00d76420b43c2f7b4870cacb712d390efe7394ad2a8245f6825
 SHA512:
-  metadata.gz: 3c217d761ea4bde8cbbdd231229a0d7e209e97b15de7756e76f4354fefb08c5f7eb5774842a239d2312feba262685166712e915d2ea1dcc50d5572809fdafe76
-  data.tar.gz: 640b40256d2b9db5a1a62093d865ea51ad5b92ce2ac3dc1ba5c332665148e54f7fd10da5675eceb7e18ba54840454fc21e31849b4ced8770fee60ea592f582cb
+  metadata.gz: 1a4e9981f2e6334b2862a073eb645b893bbebd42399f0e4b3090d98ea3d69ab0d3104bebe34f86e66b3ec2ea01b827efa28b75c2be55482de6f89a8b082b2113
+  data.tar.gz: acab89439a3655e51c3173bc30199bbccb6511567803fa16f066b26fbd6e5934f75492b6ed6383c8c59a27e397e23c781226abd439c551bc6be63c001868bc3a

data/README.md

@@ -3,7 +3,7 @@
 ## [Documentation](https://docs.rubycdp.com/docs/ferrum/introduction)
 
 <img align="right"
-     width="320" height="241"
+     width="160" height="160"
      alt="Ferrum logo"
      src="https://raw.githubusercontent.com/rubycdp/ferrum/main/logo.svg?sanitize=true">
 
@@ -15,9 +15,11 @@ protocol because Chrome allows you to do so many things that are barely
 supported by WebDriver because it should have consistent design with other
 browsers.
 
-* [Cuprite](https://github.com/rubycdp/cuprite) is a pure Ruby driver for[Capybara](https://github.com/teamcapybara/capybara) based on Ferrum.
+* [Cuprite](https://github.com/rubycdp/cuprite) is a pure Ruby driver for [Capybara](https://github.com/teamcapybara/capybara) based on Ferrum.
 * [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework based on Ferrum and Mechanize.
 
+<br clear="right">
+
 ## Install
 
 There's no official Chrome or Chromium package for Linux don't install it this

data/lib/ferrum/accessibility/ax_node.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Accessibility
+    #
+    # Represents an [AXNode](https://chromedevtools.github.io/devtools-protocol/tot/Accessibility/#type-AXNode)
+    # from the CDP Accessibility domain.
+    #
+    class AXNode
+      #
+      # @param [Hash{String => Object}] params
+      #   The parsed CDP AXNode attributes.
+      #
+      def initialize(params)
+        @params = deep_freeze(params)
+      end
+
+      # The AX node's role, e.g. `"button"`.
+      #
+      # @return [String, nil]
+      def role
+        @params.dig("role", "value")
+      end
+
+      # The AX node's accessible name.
+      #
+      # @return [String, nil]
+      def name
+        @params.dig("name", "value")
+      end
+
+      # The AX node's accessible description.
+      #
+      # @return [String, nil]
+      def description
+        @params.dig("description", "value")
+      end
+
+      # The AX node's current value.
+      #
+      # @return [String, Numeric, Boolean, nil] raw CDP AXValue.value; type varies by control
+      def value
+        @params.dig("value", "value")
+      end
+
+      # ARIA/computed properties flattened to `name => value`.
+      #
+      # @return [Hash{String => Object}]
+      def properties
+        @properties ||= Array(@params["properties"]).to_h do |property|
+          [property["name"], property.dig("value", "value")]
+        end.freeze
+      end
+
+      # Whether the node is ignored by the accessibility tree.
+      #
+      # @return [Boolean]
+      def ignored?
+        @params["ignored"] == true
+      end
+
+      # The reasons the node is ignored, if any.
+      #
+      # @return [Array, nil]
+      def ignored_reasons
+        @params["ignoredReasons"]
+      end
+
+      # The AX node's id.
+      #
+      # @return [String, nil]
+      def node_id
+        @params["nodeId"]
+      end
+
+      # The id of the DOM node this AX node is backed by.
+      #
+      # @return [Integer, nil]
+      def backend_dom_node_id
+        @params["backendDOMNodeId"]
+      end
+
+      # The ids of this AX node's children.
+      #
+      # @return [Array, nil]
+      def child_ids
+        @params["childIds"]
+      end
+
+      # The raw CDP AXNode hash.
+      #
+      # @return [Hash]
+      def to_h
+        @params
+      end
+
+      private
+
+      def deep_freeze(object)
+        case object
+        when Hash  then object.each { |key, value| deep_freeze(key.freeze) && deep_freeze(value) }
+        when Array then object.each { |value| deep_freeze(value) }
+        end
+        object.freeze
+      end
+    end
+  end
+end

data/lib/ferrum/accessibility.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "ferrum/accessibility/ax_node"
+
+module Ferrum
+  #
+  # Wraps the CDP [Accessibility](https://chromedevtools.github.io/devtools-protocol/tot/Accessibility/)
+  # domain. The query commands work without `enable`; `enable`/`disable` are
+  # provided for completeness (live AX events).
+  #
+  # @note The node-scoped methods (`node_for`, `partial_tree`, `query` with a
+  #   `node:`) issue the command against the node's owning page session. They
+  #   support same-process (same-target) iframes; nodes living in an
+  #   out-of-process iframe (OOPIF, separate CDP target) are not resolvable and
+  #   will error or return an empty result.
+  #
+  class Accessibility
+    def initialize(page)
+      @page = page
+    end
+
+    #
+    # The single non-ignored AXNode for a DOM node, or `nil`.
+    #
+    # @param [Ferrum::Node] node
+    # @return [AXNode, nil]
+    #
+    def node_for(node)
+      partial_tree(node: node).find { |ax_node| !ax_node.ignored? }
+    end
+
+    #
+    # The partial AX tree for a DOM node.
+    #
+    # @param [Ferrum::Node] node
+    # @param [Boolean] fetch_relatives
+    # @return [Array<AXNode>]
+    #
+    def partial_tree(node:, fetch_relatives: false)
+      nodes = node.page.command("Accessibility.getPartialAXTree",
+                                nodeId: node.node_id,
+                                fetchRelatives: fetch_relatives)["nodes"]
+      build(nodes)
+    end
+
+    #
+    # The full AX tree for the page.
+    #
+    # @param [Integer, nil] depth
+    # @param [String, nil] frame_id
+    # @return [Array<AXNode>]
+    #
+    def snapshot(depth: nil, frame_id: nil)
+      params = { depth: depth, frameId: frame_id }.compact
+      build(@page.command("Accessibility.getFullAXTree", **params)["nodes"])
+    end
+
+    #
+    # Query the AX tree by accessible name and/or role.
+    #
+    # @param [String, nil] name
+    # @param [String, nil] role
+    # @param [Ferrum::Node, nil] node  Scope the query to this node's subtree.
+    # @return [Array<AXNode>]
+    #
+    def query(name: nil, role: nil, node: nil)
+      page = node ? node.page : @page
+      params = { accessibleName: name, role: role }.compact
+      params[:nodeId] = node ? node.node_id : page.document_node_id
+      build(page.command("Accessibility.queryAXTree", **params)["nodes"])
+    end
+
+    #
+    # The root AXNode of the (optionally framed) document.
+    #
+    # @param [String, nil] frame_id
+    # @return [AXNode, nil]
+    #
+    def root(frame_id: nil)
+      params = { depth: 1, frameId: frame_id }.compact
+      build(@page.command("Accessibility.getFullAXTree", **params)["nodes"]).first
+    end
+
+    # Enables the Accessibility domain, activating live AX tree change events.
+    #
+    # @return [self]
+    def enable
+      @page.command("Accessibility.enable")
+      self
+    end
+
+    # Disables the Accessibility domain.
+    #
+    # @return [self]
+    def disable
+      @page.command("Accessibility.disable")
+      self
+    end
+
+    private
+
+    def build(nodes)
+      Array(nodes).map { |node| AXNode.new(node) }
+    end
+  end
+end

data/lib/ferrum/browser/binary.rb

@@ -2,23 +2,64 @@
 
 module Ferrum
   class Browser
+    #
+    # Locates an executable on the system `PATH`, mirroring what a shell's
+    # `which`/`where` would find. Used to resolve the browser (and Xvfb)
+    # binary when no explicit path is configured.
+    #
     module Binary
       module_function
 
+      #
+      # Finds the first executable path for the given command(s) on `PATH`.
+      #
+      # @param [String, Array<String>] commands
+      #   Command name(s) to look up.
+      #
+      # @return [String, nil]
+      #   Absolute path to the executable, or `nil` if none is found.
+      #
       def find(commands)
         enum(commands).first
       end
 
+      #
+      # Finds all executable paths for the given command(s) on `PATH`.
+      #
+      # @param [String, Array<String>] commands
+      #   Command name(s) to look up.
+      #
+      # @return [Array<String>]
+      #   Absolute paths to matching executables.
+      #
       def all(commands)
         enum(commands).force
       end
 
+      #
+      # Lazily enumerates executable paths for the given command(s) on `PATH`.
+      #
+      # @param [String, Array<String>] commands
+      #   Command name(s) to look up.
+      #
+      # @return [Enumerator::Lazy]
+      #   Lazy enumerator yielding matching executable paths.
+      #
       def enum(commands)
         paths, exts = prepare_paths
         cmds = Array(commands).product(paths, exts)
         lazy_find(cmds)
       end
 
+      #
+      # Directories on `PATH` and the executable extensions to try against them.
+      #
+      # @return [Array(Array<String>, Array<String>)]
+      #   The `PATH` directories, and the extensions from `PATHEXT` (plus `""`).
+      #
+      # @raise [EmptyPathError]
+      #   If `PATH` is empty.
+      #
       def prepare_paths
         exts = (ENV.key?("PATHEXT") ? ENV.fetch("PATHEXT").split(";") : []) << ""
         paths = ENV["PATH"].split(File::PATH_SEPARATOR)
@@ -27,7 +68,7 @@ module Ferrum
         [paths, exts]
       end
 
-      # rubocop:disable Style/CollectionCompact
+      # rubocop:disable-next Style/CollectionCompact
       def lazy_find(cmds)
         cmds.lazy.map do |cmd, path, ext|
           absolute_path = File.absolute_path(cmd)
@@ -40,7 +81,6 @@ module Ferrum
           cmd
         end.reject(&:nil?) # .compact isn't defined on Enumerator::Lazy
       end
-      # rubocop:enable Style/CollectionCompact
     end
   end
 end

data/lib/ferrum/browser/command.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Browser
+    #
+    # Builds the OS-level command used to spawn the browser process: resolves
+    # the executable path and merges the browser-specific required/default
+    # flags (see {Options::Base}) with any user-supplied ones into a single
+    # argument list.
+    #
     class Command
       NOT_FOUND = "Could not find an executable for the browser. Try to make " \
                   "it available on the PATH or set environment variable for " \
@@ -11,8 +17,6 @@ module Ferrum
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
       def self.build(options, user_data_dir)
         defaults = case options.browser_name
-                   when :firefox
-                     Options::Firefox.options
                    when :chrome, :opera, :edge, nil
                      Options::Chrome.options
                    else
@@ -35,14 +39,28 @@ module Ferrum
         merge_options
       end
 
+      # Whether the browser should be launched under Xvfb.
+      #
+      # @return [Boolean]
       def xvfb?
         !!options.xvfb
       end
 
+      #
+      # Command line arguments for spawning the browser process.
+      #
+      # @return [Array<String>]
+      #   The browser path followed by its CLI flags.
+      #
       def to_a
         [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
       end
 
+      #
+      # String representation of the command used to spawn the browser process.
+      #
+      # @return [String]
+      #
       def to_s
         to_a.join(" \\ \n  ")
       end

data/lib/ferrum/browser/options/base.rb

@@ -6,14 +6,29 @@ require "open3"
 module Ferrum
   class Browser
     class Options
+      #
+      # Abstract base for browser-specific default option builders. Subclasses
+      # (e.g. {Options::Chrome}) declare their own `DEFAULT_OPTIONS` and
+      # `PLATFORM_PATH` constants and implement {#merge_required}/{#merge_default}
+      # to produce the final CLI flags used by {Command}.
+      #
       class Base
         include Singleton
 
+        #
+        # Singleton instance holding the browser's default options.
+        #
+        # @return [Base]
+        #
         def self.options
           instance
         end
 
+        # The installed browser's version string, obtained by running its
+        # binary with `--version`.
+        #
         # @return [String, nil]
+        #   The version output, or `nil` if the binary can't be found/run.
         def self.version
           out, = Open3.capture2(instance.detect_path, "--version")
           out.strip
@@ -21,22 +36,73 @@ module Ferrum
           nil
         end
 
+        #
+        # Default CLI flags for the browser.
+        #
+        # @return [Hash{String => Object}]
+        #
         def to_h
           self.class::DEFAULT_OPTIONS
         end
 
+        #
+        # Default CLI flags excluding the given keys.
+        #
+        # @param [Array<String>] keys
+        #   Flag names to exclude.
+        #
+        # @return [Hash{String => Object}]
+        #
         def except(*keys)
           to_h.except(*keys)
         end
 
+        #
+        # Locates the browser executable on the system.
+        #
+        # @return [String, nil]
+        #   Absolute path to the browser binary, or `nil` if not found.
+        #
         def detect_path
           Binary.find(self.class::PLATFORM_PATH[Utils::Platform.platform_name])
         end
 
+        #
+        # Merges the CLI flags that are always required to drive the browser
+        # over CDP (e.g. remote debugging port, user data dir) into `flags`.
+        # Abstract; overridden by {Options::Chrome}.
+        #
+        # @param [Hash{String => Object}] flags
+        #   The flags accumulated so far.
+        # @param [Options] options
+        #   The user-supplied browser options.
+        # @param [String] user_data_dir
+        #   Path to the browser's user data directory.
+        #
+        # @return [Hash{String => Object}]
+        #
+        # @raise [NotImplementedError]
+        #   Always, unless overridden by a subclass.
+        #
         def merge_required(flags, options, user_data_dir)
           raise NotImplementedError
         end
 
+        #
+        # Merges this browser's default CLI flags (see {#to_h}) into `flags`,
+        # unless `options.ignore_default_browser_options` is set. Abstract;
+        # overridden by {Options::Chrome}.
+        #
+        # @param [Hash{String => Object}] flags
+        #   The flags accumulated so far.
+        # @param [Options] options
+        #   The user-supplied browser options.
+        #
+        # @return [Hash{String => Object}]
+        #
+        # @raise [NotImplementedError]
+        #   Always, unless overridden by a subclass.
+        #
         def merge_default(flags, options)
           raise NotImplementedError
         end

data/lib/ferrum/browser/options/chrome.rb

@@ -3,6 +3,12 @@
 module Ferrum
   class Browser
     class Options
+      #
+      # Chrome/Chromium-specific default flags and binary locations. Provides
+      # the CLI flags required to drive Chrome over CDP as well as the
+      # hardened defaults (disabling background networking, extensions,
+      # infobars, etc.) applied unless the user opts out.
+      #
       class Chrome < Base
         DEFAULT_OPTIONS = {
           "allow-pre-commit-input" => nil,
@@ -80,9 +86,23 @@ module Ferrum
           linux: LINUX_BIN_PATH
         }.freeze
 
+        #
+        # Merges CLI flags required for Chrome to work with CDP.
+        #
+        # @param [Hash] flags
+        #   Flags to merge required ones into.
+        #
+        # @param [Ferrum::Browser::Options] options
+        #   Browser options.
+        #
+        # @param [String] user_data_dir
+        #   Path to the browser's user data directory.
+        #
+        # @return [Hash]
+        #   Merged flags.
+        #
         def merge_required(flags, options, user_data_dir)
           flags = flags.merge("remote-debugging-port" => options.port,
-                              "remote-debugging-address" => options.host,
                               "window-size" => options.window_size&.join(","),
                               "user-data-dir" => user_data_dir)
 
@@ -94,8 +114,21 @@ module Ferrum
           flags
         end
 
+        #
+        # Merges Chrome's default flags with the given ones, unless the browser
+        # is configured to ignore default browser options.
+        #
+        # @param [Hash] flags
+        #   Flags that take precedence over the defaults.
+        #
+        # @param [Ferrum::Browser::Options] options
+        #   Browser options.
+        #
+        # @return [Hash]
+        #   Merged flags.
+        #
         def merge_default(flags, options)
-          defaults = options.headless == false ? except("headless", "disable-gpu") : DEFAULT_OPTIONS
+          defaults = options.headless == false ? except("headless", "disable-gpu") : DEFAULT_OPTIONS.dup
           defaults.delete("no-startup-window") if options.incognito == false
 
           if options.dockerize || ENV["FERRUM_CHROME_DOCKERIZE"] == "true"

data/lib/ferrum/browser/options.rb

@@ -2,12 +2,19 @@
 
 module Ferrum
   class Browser
+    #
+    # Resolves and normalizes the options hash passed to {Browser.new},
+    # applying defaults for connection settings (host, port, timeouts),
+    # window size, and other launch behavior. Used throughout the browser to
+    # look up user-configured settings.
+    #
     class Options
       BROWSER_PORT = "0"
       BROWSER_HOST = "127.0.0.1"
       WINDOW_SIZE = [1024, 768].freeze
       BASE_URL_SCHEMA = %w[http https].freeze
       DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
+      DEFAULT_PROTOCOL_TIMEOUT = ENV.fetch("FERRUM_PROTOCOL_TIMEOUT", DEFAULT_TIMEOUT).to_i
       PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
       DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
 
@@ -16,7 +23,7 @@ module Ferrum
                   :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
                   :save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
                   :ignore_default_browser_options, :xvfb, :flatten
-      attr_accessor :timeout, :default_user_agent
+      attr_accessor :timeout, :protocol_timeout, :default_user_agent
 
       def initialize(options = nil)
         @options = Hash(options&.dup)
@@ -24,13 +31,14 @@ module Ferrum
         @port = @options.fetch(:port, BROWSER_PORT)
         @host = @options.fetch(:host, BROWSER_HOST)
         @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
+        @protocol_timeout = @options.fetch(:protocol_timeout, DEFAULT_PROTOCOL_TIMEOUT)
         @window_size = @options.fetch(:window_size, WINDOW_SIZE)
         @js_errors = @options.fetch(:js_errors, false)
         @headless = @options.fetch(:headless, true)
         @incognito = @options.fetch(:incognito, true)
         @dockerize = @options.fetch(:dockerize, false)
         @flatten = @options.fetch(:flatten, true)
-        @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+        @pending_connection_errors = @options.fetch(:pending_connection_errors, false)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
         @slowmo = @options[:slowmo].to_f
 
@@ -52,16 +60,42 @@ module Ferrum
         @browser_options = @options.fetch(:browser_options, {}).freeze
       end
 
+      #
+      # Sets the base URL relative navigations are resolved against.
+      #
+      # @param [String] value
+      #   An absolute URL including scheme, e.g. `"https://example.com"`.
+      #
+      # @return [Addressable::URI]
+      #
       def base_url=(value)
         @base_url = parse_base_url(value)
       end
 
+      #
+      # JS source to preload into the browser, read from `:extensions` option.
+      #
+      # @return [Array<String>]
+      #   JS source code for each configured extension.
+      #
       def extensions
         @extensions ||= Array(@options[:extensions]).map do |extension|
           (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
         end
       end
 
+      #
+      # Validates the `:proxy` option, if given.
+      #
+      # @param [Hash, nil] options
+      #   The `:proxy` option as passed to {#initialize}.
+      #
+      # @return [Hash, nil]
+      #   The same `options`, unchanged.
+      #
+      # @raise [ArgumentError]
+      #   If `options` is not a `Hash`, or is a `Hash` without `:host` or `:port`.
+      #
       def validate_proxy(options)
         return unless options
 
@@ -74,6 +108,11 @@ module Ferrum
         options
       end
 
+      #
+      # Raw options hash used to initialize the browser.
+      #
+      # @return [Hash{Symbol => Object}]
+      #
       def to_h
         @options
       end