ferrum 0.17.2 → 0.18.0

data/lib/ferrum/proxy.rb

@@ -1,11 +1,34 @@
 # frozen_string_literal: true
 
 require "tempfile"
-require "webrick"
-require "webrick/httpproxy"
+
+begin
+  require "webrick"
+  require "webrick/httpproxy"
+rescue LoadError
+  warn("Please add webrick to your Gemfile to use Ferrum proxy")
+  raise
+end
 
 module Ferrum
+  #
+  # A local WEBrick-based proxy server, useful for injecting HTTP Basic-Auth
+  # credentials into requests or forwarding to (and rotating between)
+  # upstream proxies, cases the browser's own proxy support can't handle
+  # directly.
+  #
+  # @note Requires the `webrick` gem, which isn't a hard dependency of
+  #   Ferrum; add it to your Gemfile to use this class.
+  #
   class Proxy
+    #
+    # Builds a new proxy server and starts it.
+    #
+    # @param [Hash] args
+    #   Keyword arguments forwarded to {#initialize}.
+    #
+    # @return [Proxy]
+    #
     def self.start(**args)
       new(**args).tap(&:start)
     end
@@ -20,6 +43,11 @@ module Ferrum
       @password = password
     end
 
+    #
+    # Starts the WEBrick proxy server.
+    #
+    # @return [void]
+    #
     def start
       options = {
         ProxyURI: nil, ServerType: Thread,
@@ -45,12 +73,34 @@ module Ferrum
       @port = @server.config[:Port]
     end
 
+    #
+    # Changes the upstream proxy the server forwards connections to.
+    #
+    # @param [String] host
+    #   Address of the upstream proxy.
+    #
+    # @param [Integer] port
+    #   Port of the upstream proxy.
+    #
+    # @param [String, nil] user
+    #   Username for upstream proxy authentication.
+    #
+    # @param [String, nil] password
+    #   Password for upstream proxy authentication.
+    #
+    # @return [void]
+    #
     def rotate(host:, port:, user: nil, password: nil)
       credentials = "#{user}:#{password}@" if user && password
       proxy_uri = "schema://#{credentials}#{host}:#{port}"
       @server.config[:ProxyURI] = URI.parse(proxy_uri)
     end
 
+    #
+    # Stops the proxy server and removes the htpasswd file.
+    #
+    # @return [void]
+    #
     def stop
       @file&.close(true)
       @server.shutdown

data/lib/ferrum/rgba.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents an RGBA color, validating that the red/green/blue components
+  # are integers between 0 and 255 and that alpha is a float between 0.0 and
+  # 1.0. Used e.g. as the background color option for screenshots.
+  #
   class RGBA
     def initialize(red, green, blue, alpha)
       self.red = red
@@ -11,6 +16,11 @@ module Ferrum
       validate
     end
 
+    #
+    # Converts the color to a Hash.
+    #
+    # @return [Hash{Symbol => Integer, Float}]
+    #
     def to_h
       { r: red, g: green, b: blue, a: alpha }
     end

data/lib/ferrum/target.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents a CDP target, e.g. a page, iframe, dedicated/shared worker or
+  # service worker. Wraps the raw `targetInfo` params and lazily connects to
+  # the target as a {Page} or {Worker} over its own {Client}/{SessionClient}
+  # session, so a `Target` can exist without being connected to yet.
+  #
   class Target
     NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
 
@@ -13,70 +19,187 @@ module Ferrum
 
     def initialize(browser_client, session_id = nil, params = nil)
       @page = nil
+      @worker = nil
       @session_id = session_id
       @params = params
       @browser_client = browser_client
       @options = browser_client.options
     end
 
+    # Merges freshly received CDP target info into this target's params,
+    # e.g. on `Target.targetInfoChanged`.
+    #
+    # @param [Hash] params
+    #
+    # @return [Hash]
     def update(params)
       @params.merge!(params)
     end
 
+    # Whether this target has already been connected to as a {Page} or
+    # {Worker}.
+    #
+    # @return [Boolean]
     def connected?
-      !!@page
+      !!@page || !!@worker
     end
 
+    # Connects to and returns this target's {Page}.
+    #
+    # @return [Page]
     def page
       @page ||= build_page
     end
 
+    # Connects to and returns this target's {Worker}.
+    #
+    # @return [Worker]
+    def worker
+      @worker ||= build_worker
+    end
+
+    # The client connected to this target's own session (or a dedicated
+    # WebSocket connection if `options.flatten` is disabled).
+    #
+    # @return [Client, SessionClient]
     def client
       @client ||= build_client
     end
 
+    # Builds a {Page} for this target. Called lazily by `page`, or eagerly
+    # by {Context#create_page}.
+    #
+    # @param [Hash] options
+    #
+    # @return [Page]
     def build_page(**options)
       maybe_sleep_if_new_window
       Page.new(client, context_id: context_id, target_id: id, **options)
     end
 
+    # Builds a {Worker} for this target. Called lazily by {#worker}.
+    #
+    # @return [Worker]
+    def build_worker
+      Worker.new(client, target_id: id, url: url)
+    end
+
+    # Closes the WebSocket connection, without closing the target itself
+    # in the browser.
+    #
+    # @return [void]
+    def close_connection
+      @page&.close_connection
+      @worker&.close_connection
+    end
+
+    # The target's id.
+    #
+    # @return [String]
     def id
       @params["targetId"]
     end
 
+    # The target's type, e.g. `"page"`, `"iframe"`, `"worker"`,
+    # `"shared_worker"`, `"service_worker"`.
+    #
+    # @return [String]
     def type
       @params["type"]
     end
 
+    # The target's title.
+    #
+    # @return [String]
     def title
       @params["title"]
     end
 
+    # The target's URL.
+    #
+    # @return [String]
     def url
       @params["url"]
     end
 
+    # The id of the target that opened this one, set only for
+    # windows/tabs opened via `window.open`/links/etc.
+    #
+    # @return [String, nil]
     def opener_id
       @params["openerId"]
     end
 
+    # The id of the target that spawned this one, set for iframes and
+    # workers. Unlike `opener_id`, which is only set for windows/tabs opened
+    # via `window.open`/links/etc.
+    def parent_id
+      @params["parentId"]
+    end
+
+    # The id of the browser context this target belongs to.
+    #
+    # @return [String, nil]
     def context_id
       @params["browserContextId"]
     end
 
+    # Whether this target is a window/tab, i.e. was opened via
+    # `window.open`/a link/etc. and thus has an {#opener_id}.
+    #
+    # @return [Boolean]
     def window?
       !!opener_id
     end
 
+    # Whether this target is an iframe.
+    #
+    # @return [Boolean]
     def iframe?
       type == "iframe"
     end
 
+    # Whether this target is a page.
+    #
+    # @return [Boolean]
+    def page?
+      type == "page"
+    end
+
+    # Whether this target is a dedicated worker.
+    #
+    # @return [Boolean]
+    def worker?
+      type == "worker"
+    end
+
+    # Whether this target is a shared worker.
+    #
+    # @return [Boolean]
+    def shared_worker?
+      type == "shared_worker"
+    end
+
+    # Whether this target is a service worker.
+    #
+    # @return [Boolean]
+    def service_worker?
+      type == "service_worker"
+    end
+
+    # Chrome fires no events for a newly opened window, so we sleep a bit
+    # to give it a chance to load before connecting.
+    #
+    # @return [void]
     def maybe_sleep_if_new_window
       # Dirty hack because new window doesn't have events at all
       sleep(NEW_WINDOW_WAIT) if window?
     end
 
+    # Sends a CDP command through {#client}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
     def command(...)
       client.command(...)
     end

data/lib/ferrum/utils/attempt.rb

@@ -2,9 +2,29 @@
 
 module Ferrum
   module Utils
+    #
+    # A retry-with-backoff helper for re-running a block a fixed number of
+    # times when it raises one of a given set of exceptions, sleeping
+    # between attempts.
+    #
     module Attempt
       module_function
 
+      #
+      # Retries the block up to `max` times when one of `errors` is raised,
+      # sleeping `wait` seconds between attempts.
+      #
+      # @param [Array<Class>, Class] errors
+      #   Exception classes that trigger a retry.
+      #
+      # @param [Integer] max
+      #   Maximum number of attempts.
+      #
+      # @param [Numeric] wait
+      #   Seconds to sleep between attempts.
+      #
+      # @return [Object]
+      #
       def with_retry(errors:, max:, wait:)
         attempts ||= 1
         yield

data/lib/ferrum/utils/elapsed_time.rb

@@ -2,25 +2,63 @@
 
 module Ferrum
   module Utils
+    #
+    # A monotonic-clock helper for tracking elapsed time and checking
+    # timeouts, backed by `Concurrent.monotonic_time`.
+    #
     module ElapsedTime
       module_function
 
+      #
+      # Sets the start point to the current monotonic time unless already set.
+      #
+      # @return [Float]
+      #
       def start
         @start ||= monotonic_time
       end
 
+      #
+      # Resets the start point to the current monotonic time.
+      #
+      # @return [Float]
+      #
       def reset
         @start = monotonic_time
       end
 
+      #
+      # Returns the time elapsed since `start` (or since {.start}/{.reset} was called).
+      #
+      # @param [Float, nil] start
+      #   Monotonic time to measure from, defaults to the stored start point.
+      #
+      # @return [Float]
+      #
       def elapsed_time(start = nil)
         monotonic_time - (start || @start)
       end
 
+      #
+      # Returns the current monotonic clock time in seconds.
+      #
+      # @return [Float]
+      #
       def monotonic_time
         Concurrent.monotonic_time
       end
 
+      #
+      # Whether more than `timeout` seconds have elapsed since `start`.
+      #
+      # @param [Float] start
+      #   Monotonic time to measure from.
+      #
+      # @param [Float, Integer] timeout
+      #   The timeout, in seconds.
+      #
+      # @return [Boolean]
+      #
       def timeout?(start, timeout)
         elapsed_time(start) > timeout
       end

data/lib/ferrum/utils/event.rb

@@ -2,11 +2,25 @@
 
 module Ferrum
   module Utils
+    #
+    # A {Concurrent::Event} subclass that additionally tracks how many
+    # times it has been reset, via {#iteration}.
+    #
     class Event < Concurrent::Event
+      #
+      # Number of times the event has been reset.
+      #
+      # @return [Integer]
+      #
       def iteration
         synchronize { @iteration }
       end
 
+      #
+      # Marks the event as unset and increments the iteration counter.
+      #
+      # @return [Integer]
+      #
       def reset
         synchronize do
           @iteration += 1

data/lib/ferrum/utils/platform.rb

@@ -2,9 +2,18 @@
 
 module Ferrum
   module Utils
+    #
+    # OS and Ruby engine detection helpers, used to pick the right browser
+    # path/flags and behavior for the current platform.
+    #
     module Platform
       module_function
 
+      #
+      # Detects the current platform.
+      #
+      # @return [:mac, :windows, :linux]
+      #
       def platform_name
         return :mac if mac?
         return :windows if windows?
@@ -12,18 +21,30 @@ module Ferrum
         :linux
       end
 
+      # Whether the current platform is Windows.
+      #
+      # @return [Boolean]
       def windows?
         RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
       end
 
+      # Whether the current platform is macOS.
+      #
+      # @return [Boolean]
       def mac?
         RbConfig::CONFIG["host_os"] =~ /darwin/
       end
 
+      # Whether the current platform is macOS running on Apple Silicon (arm64).
+      #
+      # @return [Boolean]
       def mac_arm?
         mac? && RbConfig::CONFIG["host_cpu"] =~ /arm/
       end
 
+      # Whether the current Ruby engine is MRI (as opposed to e.g. JRuby, TruffleRuby).
+      #
+      # @return [Boolean]
       def mri?
         defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
       end

data/lib/ferrum/utils/thread.rb

@@ -2,9 +2,21 @@
 
 module Ferrum
   module Utils
+    #
+    # A helper for spawning threads with consistent exception-handling
+    # behavior (abort-on-exception and report-on-exception).
+    #
     module Thread
       module_function
 
+      #
+      # Spawns a new thread running the given block.
+      #
+      # @param [Boolean] abort_on_exception
+      #   Whether the thread aborts the process if it raises an unhandled exception.
+      #
+      # @return [Thread]
+      #
       def spawn(abort_on_exception: true)
         ::Thread.new(abort_on_exception) do |whether_abort_on_exception|
           ::Thread.current.abort_on_exception = whether_abort_on_exception

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.17.2"
+  VERSION = "0.18.0"
 end

data/lib/ferrum/worker.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "ferrum/frame/runtime"
+require "ferrum/interceptable"
+
+module Ferrum
+  # A dedicated or shared Worker spawned by a page. Unlike {Page} it has no
+  # DOM, frames, mouse/keyboard, or navigation history -- just a single
+  # global execution context and its own network activity.
+  class Worker
+    include Frame::Runtime
+    include Interceptable
+
+    # Client connection.
+    #
+    # @return [Client, SessionClient]
+    attr_reader :client
+
+    attr_reader :target_id, :url
+
+    def initialize(client, target_id:, url:)
+      @client = client
+      @target_id = target_id
+      @url = url
+      @options = client.options
+      @page = self
+      @execution_id = Concurrent::MVar.new
+      @network = Network.new(self)
+
+      subscribe
+      prepare
+    end
+
+    # Network object.
+    #
+    # @return [Network]
+    attr_reader :network
+
+    # How long to wait for CDP responses and JS evaluation to complete.
+    #
+    # @return [Numeric]
+    def timeout
+      @options.timeout
+    end
+
+    # Sends a CDP command through {#client}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
+    def command(...)
+      client.command(...)
+    end
+
+    # Workers have a single execution context and no navigable document, so
+    # there's nothing for {Network} to check requests against.
+    def main_frame
+      nil
+    end
+
+    # Closes the target in the browser, and its connection.
+    #
+    # @return [Boolean]
+    def close
+      client.command("Target.closeTarget", async: true, targetId: target_id)
+      close_connection
+
+      true
+    end
+
+    # Closes the WebSocket connection only, without asking the browser to
+    # close the underlying target. Kept separate from {#close} so a whole
+    # context can be dropped (the browser closes its targets anyway) without
+    # every worker having to be closed one by one.
+    #
+    # @return [void]
+    def close_connection
+      client&.close
+    end
+
+    # Debug representation of the worker.
+    #
+    # @return [String]
+    def inspect
+      "#<#{self.class} @target_id=#{@target_id.inspect} @url=#{@url.inspect}>"
+    end
+
+    private
+
+    def subscribe
+      network.subscribe
+
+      on("Runtime.executionContextCreated") do |params|
+        self.execution_id = params.dig("context", "id")
+      end
+
+      return unless @options.js_errors
+
+      on("Runtime.exceptionThrown") do |params|
+        # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+        Thread.main.raise JavaScriptError, params["exceptionDetails"]
+      end
+    end
+
+    def prepare
+      command("Runtime.enable")
+      command("Network.enable")
+      command("Runtime.runIfWaitingForDebugger")
+    end
+
+    def execution_id!
+      value = @execution_id.borrow(timeout, &:itself)
+      raise NoExecutionContextError if value.instance_of?(Object)
+
+      value
+    end
+
+    def execution_id=(value)
+      if value.nil?
+        @execution_id.try_take!
+      else
+        @execution_id.try_put!(value)
+      end
+    end
+  end
+end

data/lib/ferrum.rb

@@ -10,5 +10,12 @@ require "ferrum/errors"
 require "ferrum/browser"
 require "ferrum/node"
 
+#
+# Ferrum is a pure Ruby driver for headless Chrome and Firefox. It talks
+# directly to the browser over the Chrome DevTools Protocol (CDP), so it
+# doesn't depend on Selenium, WebDriver or any other third party gem.
+#
+# {Ferrum::Browser} is the entry point most applications start from.
+#
 module Ferrum
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 0.18.0
 platform: ruby
 authors:
 - Dmitry Vorotilin
@@ -51,20 +51,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: webrick
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: websocket-driver
   requirement: !ruby/object:Gem::Requirement
@@ -89,6 +75,8 @@ files:
 - LICENSE
 - README.md
 - lib/ferrum.rb
+- lib/ferrum/accessibility.rb
+- lib/ferrum/accessibility/ax_node.rb
 - lib/ferrum/browser.rb
 - lib/ferrum/browser/binary.rb
 - lib/ferrum/browser/command.rb
@@ -113,6 +101,7 @@ files:
 - lib/ferrum/frame/dom.rb
 - lib/ferrum/frame/runtime.rb
 - lib/ferrum/headers.rb
+- lib/ferrum/interceptable.rb
 - lib/ferrum/keyboard.json
 - lib/ferrum/keyboard.rb
 - lib/ferrum/mouse.rb
@@ -141,6 +130,7 @@ files:
 - lib/ferrum/utils/platform.rb
 - lib/ferrum/utils/thread.rb
 - lib/ferrum/version.rb
+- lib/ferrum/worker.rb
 homepage: https://github.com/rubycdp/ferrum
 licenses:
 - MIT
@@ -165,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Ruby headless Chrome driver
 test_files: []