ferrum 0.17.1 → 0.18.0

data/lib/ferrum/browser/options.rb

@@ -2,21 +2,28 @@
 
 module Ferrum
   class Browser
+    #
+    # Resolves and normalizes the options hash passed to {Browser.new},
+    # applying defaults for connection settings (host, port, timeouts),
+    # window size, and other launch behavior. Used throughout the browser to
+    # look up user-configured settings.
+    #
     class Options
       BROWSER_PORT = "0"
       BROWSER_HOST = "127.0.0.1"
       WINDOW_SIZE = [1024, 768].freeze
       BASE_URL_SCHEMA = %w[http https].freeze
       DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
+      DEFAULT_PROTOCOL_TIMEOUT = ENV.fetch("FERRUM_PROTOCOL_TIMEOUT", DEFAULT_TIMEOUT).to_i
       PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
       DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
 
       attr_reader :window_size, :logger, :ws_max_receive_size,
                   :js_errors, :base_url, :slowmo, :pending_connection_errors,
                   :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
-                  :save_path, :proxy, :port, :host, :headless, :incognito, :browser_options,
+                  :save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
                   :ignore_default_browser_options, :xvfb, :flatten
-      attr_accessor :timeout, :default_user_agent
+      attr_accessor :timeout, :protocol_timeout, :default_user_agent
 
       def initialize(options = nil)
         @options = Hash(options&.dup)
@@ -24,12 +31,14 @@ module Ferrum
         @port = @options.fetch(:port, BROWSER_PORT)
         @host = @options.fetch(:host, BROWSER_HOST)
         @timeout = @options.fetch(:timeout, DEFAULT_TIMEOUT)
+        @protocol_timeout = @options.fetch(:protocol_timeout, DEFAULT_PROTOCOL_TIMEOUT)
         @window_size = @options.fetch(:window_size, WINDOW_SIZE)
         @js_errors = @options.fetch(:js_errors, false)
         @headless = @options.fetch(:headless, true)
         @incognito = @options.fetch(:incognito, true)
+        @dockerize = @options.fetch(:dockerize, false)
         @flatten = @options.fetch(:flatten, true)
-        @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
+        @pending_connection_errors = @options.fetch(:pending_connection_errors, false)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
         @slowmo = @options[:slowmo].to_f
 
@@ -51,16 +60,42 @@ module Ferrum
         @browser_options = @options.fetch(:browser_options, {}).freeze
       end
 
+      #
+      # Sets the base URL relative navigations are resolved against.
+      #
+      # @param [String] value
+      #   An absolute URL including scheme, e.g. `"https://example.com"`.
+      #
+      # @return [Addressable::URI]
+      #
       def base_url=(value)
         @base_url = parse_base_url(value)
       end
 
+      #
+      # JS source to preload into the browser, read from `:extensions` option.
+      #
+      # @return [Array<String>]
+      #   JS source code for each configured extension.
+      #
       def extensions
         @extensions ||= Array(@options[:extensions]).map do |extension|
           (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
         end
       end
 
+      #
+      # Validates the `:proxy` option, if given.
+      #
+      # @param [Hash, nil] options
+      #   The `:proxy` option as passed to {#initialize}.
+      #
+      # @return [Hash, nil]
+      #   The same `options`, unchanged.
+      #
+      # @raise [ArgumentError]
+      #   If `options` is not a `Hash`, or is a `Hash` without `:host` or `:port`.
+      #
       def validate_proxy(options)
         return unless options
 
@@ -73,6 +108,11 @@ module Ferrum
         options
       end
 
+      #
+      # Raw options hash used to initialize the browser.
+      #
+      # @return [Hash{Symbol => Object}]
+      #
       def to_h
         @options
       end

data/lib/ferrum/browser/process.rb

@@ -14,21 +14,40 @@ require "ferrum/utils/platform"
 
 module Ferrum
   class Browser
+    #
+    # Spawns and manages the lifecycle of the browser OS process: builds the
+    # launch {Command}, starts it (optionally under {Xvfb} for headful mode),
+    # waits for the CDP WebSocket endpoint to become available, and handles
+    # stopping/restarting the process and cleaning up its user data directory.
+    #
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
 
-      attr_reader :host, :port, :ws_url, :pid, :command,
-                  :default_user_agent, :browser_version, :protocol_version,
-                  :v8_version, :webkit_version, :xvfb
-
       extend Forwardable
+
       delegate path: :command
 
+      #
+      # Builds and starts a new browser process.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Process]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
 
+      #
+      # Builds a finalizer proc that kills the process with the given pid.
+      #
+      # @param [Integer] pid
+      #   Process id to kill.
+      #
+      # @return [Proc]
+      #
       def self.process_killer(pid)
         proc do
           if Utils::Platform.windows?
@@ -51,6 +70,14 @@ module Ferrum
         end
       end
 
+      #
+      # Builds a finalizer proc that removes the directory at the given path.
+      #
+      # @param [String] path
+      #   Directory to remove.
+      #
+      # @return [Proc]
+      #
       def self.directory_remover(path)
         proc {
           begin
@@ -61,6 +88,10 @@ module Ferrum
         }
       end
 
+      attr_reader :host, :port, :ws_url, :pid, :command,
+                  :default_user_agent, :browser_version, :protocol_version,
+                  :v8_version, :webkit_version, :xvfb
+
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
 
@@ -82,6 +113,11 @@ module Ferrum
         @command = Command.build(options, tmpdir)
       end
 
+      #
+      # Spawns the browser process and waits for it to become reachable over CDP.
+      #
+      # @return [void]
+      #
       def start
         # Don't do anything as browser is already running as external process.
         return if ws_url
@@ -108,6 +144,12 @@ module Ferrum
         end
       end
 
+      #
+      # Kills the browser process (and Xvfb, if running) and removes the user
+      # data directory.
+      #
+      # @return [void]
+      #
       def stop
         if @pid
           kill(@pid)
@@ -119,11 +161,21 @@ module Ferrum
         ObjectSpace.undefine_finalizer(self)
       end
 
+      #
+      # Stops and starts the browser process again.
+      #
+      # @return [void]
+      #
       def restart
         stop
         start
       end
 
+      #
+      # Custom inspection that omits noisy internal command details.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@user_data_dir=#{@user_data_dir.inspect} " \

data/lib/ferrum/browser/xvfb.rb

@@ -2,10 +2,24 @@
 
 module Ferrum
   class Browser
+    #
+    # Manages an Xvfb virtual display server, letting a "headful" browser run
+    # in environments without a real display (e.g. CI). Spawned by {Process}
+    # when the `:xvfb` option is set, and exposes the `DISPLAY` environment
+    # variable the browser process needs to attach to it.
+    #
     class Xvfb
       NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
                   "it with your package manager"
 
+      #
+      # Builds and starts a new Xvfb instance.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Xvfb]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
@@ -20,11 +34,21 @@ module Ferrum
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end
 
+      #
+      # Spawns the Xvfb process on the configured display.
+      #
+      # @return [void]
+      #
       def start
         @pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
         ::Process.detach(@pid)
       end
 
+      #
+      # Environment variables needed to point the browser at this Xvfb display.
+      #
+      # @return [Hash{String => String}]
+      #
       def to_env
         { "DISPLAY" => ":#{display_id}" }
       end

data/lib/ferrum/browser.rb

@@ -3,7 +3,7 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
-require "ferrum/proxy"
+require "ferrum/worker"
 require "ferrum/client"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
@@ -13,14 +13,25 @@ require "ferrum/browser/binary"
 require "ferrum/browser/version_info"
 
 module Ferrum
+  #
+  # The main entry point of the library. Instantiating it spawns (or connects
+  # to) a browser process and opens a CDP connection to it, exposing a single
+  # default {Page} that most of the top-level methods (`go_to`, `at_css`,
+  # `screenshot`, etc.) are delegated to for convenience.
+  #
+  # @note A `Browser` can host multiple {Context}s (like incognito profiles)
+  #   and multiple {Page}s within each of them, reachable through {#contexts}
+  #   and {#create_page}.
+  #
   class Browser
     extend Forwardable
+
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
                 body doctype content=
-                headers cookies network downloads
+                headers cookies network accessibility downloads
                 mouse keyboard
                 screenshot pdf mhtml viewport_size device_pixel_ratio
                 start_screencast stop_screencast
@@ -33,7 +44,8 @@ module Ferrum
 
     attr_reader :client, :process, :contexts, :options
 
-    delegate %i[timeout timeout= base_url base_url= default_user_agent default_user_agent= extensions] => :options
+    delegate %i[timeout timeout= protocol_timeout protocol_timeout=
+                base_url base_url= default_user_agent default_user_agent= extensions] => :options
     delegate %i[command] => :client
 
     #
@@ -48,6 +60,9 @@ module Ferrum
     # @option options [Boolean] :incognito (true)
     #   Create an incognito profile for the browser startup window.
     #
+    # @option options [Boolean] :dockerize (false)
+    #   Add CLI flags to a browser to run in a container.
+    #
     # @option options [Boolean] :xvfb (false)
     #   Run browser in a virtual framebuffer.
     #
@@ -66,13 +81,18 @@ module Ferrum
     #   When present, debug output is written to this object.
     #
     # @option options [Integer, Float] :slowmo
-    #   Set a delay in seconds to wait before sending command.
-    #   Useful companion of headless option, so that you have time to see
+    #   Set a delay in seconds to wait before sending a command.
+    #   Useful companion of a headless option, so that you have time to see
     #   changes.
     #
     # @option options [Numeric] :timeout (5)
-    #   The number of seconds we'll wait for a response when communicating with
-    #   browser.
+    #   The number of seconds we'll wait for a response when communicating
+    #   with the browser: navigations, JS evaluation, DOM queries, dispatching input, etc.
+    #
+    # @option options [Numeric] :protocol_timeout (5)
+    #   The number of seconds we'll wait for an individual internal CDP
+    #   bookkeeping call to respond, e.g. `Target.createTarget`,
+    #   `Target.attachToTarget`. These normally resolve in milliseconds.
     #
     # @option options [Boolean] :js_errors
     #   When true, JavaScript errors get re-raised in Ruby.
@@ -168,7 +188,7 @@ module Ferrum
     ensure
       if block_given?
         page&.close
-        context.dispose if new_context
+        context&.dispose if new_context
       end
     end
 
@@ -207,11 +227,17 @@ module Ferrum
       contexts.reset
     end
 
+    #
+    # Restarts the browser process, keeping the same options.
+    #
     def restart
       quit
       start
     end
 
+    #
+    # Terminates the browser process and closes the client connection.
+    #
     def quit
       return unless @client
 

data/lib/ferrum/client/subscriber.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Client
+    #
+    # Dispatches incoming CDP events to registered callbacks. Messages are
+    # queued and processed on dedicated threads, with `Fetch.requestPaused`
+    # and `Fetch.authRequired` given priority so request interception isn't
+    # delayed behind other events.
+    #
     class Subscriber
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
@@ -13,6 +19,16 @@ module Ferrum
         start
       end
 
+      #
+      # Enqueues an incoming CDP message for dispatch to subscribers.
+      # `Fetch.requestPaused`/`Fetch.authRequired` messages jump the regular
+      # queue so request interception isn't delayed behind other events.
+      #
+      # @param [Hash] message
+      #   The raw CDP message, as parsed from the websocket.
+      #
+      # @return [void]
+      #
       def <<(message)
         if INTERRUPTIONS.include?(message["method"])
           @priority.push(message)
@@ -21,26 +37,68 @@ module Ferrum
         end
       end
 
+      #
+      # Registers a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Integer]
+      #   The callback's index within the event's callback list, used to
+      #   unsubscribe via {#off}.
+      #
       def on(event, &block)
         @on[event] ||= Concurrent::Array.new
         @on[event] << block
         @on[event].index(block)
       end
 
+      #
+      # Unregisters a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @param [Integer] id
+      #   The callback's index, as returned by {#on}.
+      #
+      # @return [Boolean]
+      #
       def off(event, id)
         @on[event].delete_at(id)
         true
       end
 
+      #
+      # Whether there's at least one callback registered for the event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Boolean]
+      #
       def subscribed?(event)
         @on.key?(event)
       end
 
+      #
+      # Stops the regular and priority dispatch threads.
+      #
+      # @return [void]
+      #
       def close
         @regular_thread&.kill
         @priority_thread&.kill
       end
 
+      #
+      # Removes all callbacks registered for a given session.
+      #
+      # @param [String] session_id
+      #   The session id to match against registered event keys.
+      #
+      # @return [void]
+      #
       def clear(session_id:)
         @on.delete_if { |k, _| k.match?(session_id) }
       end

data/lib/ferrum/client/web_socket.rb

@@ -6,6 +6,12 @@ require "websocket/driver"
 
 module Ferrum
   class Client
+    #
+    # Low-level WebSocket connection to the browser's CDP endpoint. Opens
+    # the raw TCP/TLS socket, drives the `websocket-driver` handshake and
+    # framing, and exposes a queue of parsed incoming messages alongside
+    # methods to send commands and close the connection.
+    #
     class WebSocket
       WEBSOCKET_BUG_SLEEP = 0.05
       DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
@@ -19,7 +25,7 @@ module Ferrum
         uri     = URI.parse(@url)
         port    = uri.port || DEFAULT_PORTS[uri.scheme]
 
-        if port == 443
+        if port == 443 || url.scheme == "wss"
           tcp = TCPSocket.new(uri.host, port)
           ssl_context = OpenSSL::SSL::SSLContext.new
           @sock = OpenSSL::SSL::SSLSocket.new(tcp, ssl_context)
@@ -30,7 +36,10 @@ module Ferrum
         end
 
         max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
-        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        @driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        # websocket-driver holds no locks and is called from many threads: commands from
+        # callers, pong/close replies from the reader. One lock keeps frames from interleaving.
+        @driver_mutex = Mutex.new
         @messages = Queue.new
 
         @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
@@ -41,22 +50,28 @@ module Ferrum
 
         start
 
-        @driver.start
+        @driver_mutex.synchronize { @driver.start }
       end
 
+      #
+      # Handles the driver's `:open` event.
+      #
+      # @return [void]
+      #
       def on_open(_event)
         # https://github.com/faye/websocket-driver-ruby/issues/46
         sleep(WEBSOCKET_BUG_SLEEP)
       end
 
+      #
+      # Handles the driver's `:message` event: parses the incoming frame
+      # as JSON and pushes it onto {#messages}. Malformed payloads are
+      # dropped rather than raised, to avoid crashing the reader thread.
+      #
+      # @return [void]
+      #
       def on_message(event)
         data = safely_parse_json(event.data)
-        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
-        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
-        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
-        # thread and the application.
-        @messages.push(data) if data
-
         output = event.data
         if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
           @screenshot_commands.delete(data&.dig("id"))
@@ -64,30 +79,66 @@ module Ferrum
         end
 
         @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
+
+        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
+        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
+        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
+        # thread and the application.
+        @messages.push(data) if data
       end
 
+      #
+      # Handles the driver's `:close` event: closes the message queue and
+      # underlying socket, then kills the reader thread.
+      #
+      # @return [void]
+      #
       def on_close(_event)
         @messages.close
         @sock.close
         @thread.kill
       end
 
+      #
+      # Serializes a CDP command to JSON and sends it as a websocket text
+      # frame.
+      #
+      # @param [Hash] data
+      #   The message to send, must include an `:id` key.
+      #
+      # @return [void]
+      #
       def send_message(data)
         @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
-        @driver.text(json)
+        @driver_mutex.synchronize { @driver.text(json) }
         @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
+      #
+      # Writes raw bytes to the underlying socket. Called by
+      # `websocket-driver` to emit frames. Closes {#messages} instead of
+      # raising if the connection has already been torn down.
+      #
+      # @param [String] data
+      #   The raw bytes to write.
+      #
+      # @return [void]
+      #
       def write(data)
         @sock.write(data)
       rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
         @messages.close
       end
 
+      #
+      # Closes the websocket connection by sending a close frame.
+      #
+      # @return [void]
+      #
       def close
-        @driver.close
+        @driver_mutex.synchronize { @driver.close }
       end
 
       private
@@ -98,7 +149,7 @@ module Ferrum
             data = @sock.readpartial(512)
             break unless data
 
-            @driver.parse(data)
+            @driver_mutex.synchronize { @driver.parse(data) }
           end
         rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
           @messages.close