ferrum 0.17.2 → 0.18.0

data/lib/ferrum/browser/process.rb

@@ -14,6 +14,12 @@ require "ferrum/utils/platform"
 
 module Ferrum
   class Browser
+    #
+    # Spawns and manages the lifecycle of the browser OS process: builds the
+    # launch {Command}, starts it (optionally under {Xvfb} for headful mode),
+    # waits for the CDP WebSocket endpoint to become available, and handles
+    # stopping/restarting the process and cleaning up its user data directory.
+    #
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
@@ -22,10 +28,26 @@ module Ferrum
 
       delegate path: :command
 
+      #
+      # Builds and starts a new browser process.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Process]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
 
+      #
+      # Builds a finalizer proc that kills the process with the given pid.
+      #
+      # @param [Integer] pid
+      #   Process id to kill.
+      #
+      # @return [Proc]
+      #
       def self.process_killer(pid)
         proc do
           if Utils::Platform.windows?
@@ -48,6 +70,14 @@ module Ferrum
         end
       end
 
+      #
+      # Builds a finalizer proc that removes the directory at the given path.
+      #
+      # @param [String] path
+      #   Directory to remove.
+      #
+      # @return [Proc]
+      #
       def self.directory_remover(path)
         proc {
           begin
@@ -83,6 +113,11 @@ module Ferrum
         @command = Command.build(options, tmpdir)
       end
 
+      #
+      # Spawns the browser process and waits for it to become reachable over CDP.
+      #
+      # @return [void]
+      #
       def start
         # Don't do anything as browser is already running as external process.
         return if ws_url
@@ -109,6 +144,12 @@ module Ferrum
         end
       end
 
+      #
+      # Kills the browser process (and Xvfb, if running) and removes the user
+      # data directory.
+      #
+      # @return [void]
+      #
       def stop
         if @pid
           kill(@pid)
@@ -120,11 +161,21 @@ module Ferrum
         ObjectSpace.undefine_finalizer(self)
       end
 
+      #
+      # Stops and starts the browser process again.
+      #
+      # @return [void]
+      #
       def restart
         stop
         start
       end
 
+      #
+      # Custom inspection that omits noisy internal command details.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@user_data_dir=#{@user_data_dir.inspect} " \

data/lib/ferrum/browser/xvfb.rb

@@ -2,10 +2,24 @@
 
 module Ferrum
   class Browser
+    #
+    # Manages an Xvfb virtual display server, letting a "headful" browser run
+    # in environments without a real display (e.g. CI). Spawned by {Process}
+    # when the `:xvfb` option is set, and exposes the `DISPLAY` environment
+    # variable the browser process needs to attach to it.
+    #
     class Xvfb
       NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
                   "it with your package manager"
 
+      #
+      # Builds and starts a new Xvfb instance.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Xvfb]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
@@ -20,11 +34,21 @@ module Ferrum
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end
 
+      #
+      # Spawns the Xvfb process on the configured display.
+      #
+      # @return [void]
+      #
       def start
         @pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
         ::Process.detach(@pid)
       end
 
+      #
+      # Environment variables needed to point the browser at this Xvfb display.
+      #
+      # @return [Hash{String => String}]
+      #
       def to_env
         { "DISPLAY" => ":#{display_id}" }
       end

data/lib/ferrum/browser.rb

@@ -3,7 +3,7 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
-require "ferrum/proxy"
+require "ferrum/worker"
 require "ferrum/client"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
@@ -13,15 +13,25 @@ require "ferrum/browser/binary"
 require "ferrum/browser/version_info"
 
 module Ferrum
+  #
+  # The main entry point of the library. Instantiating it spawns (or connects
+  # to) a browser process and opens a CDP connection to it, exposing a single
+  # default {Page} that most of the top-level methods (`go_to`, `at_css`,
+  # `screenshot`, etc.) are delegated to for convenience.
+  #
+  # @note A `Browser` can host multiple {Context}s (like incognito profiles)
+  #   and multiple {Page}s within each of them, reachable through {#contexts}
+  #   and {#create_page}.
+  #
   class Browser
     extend Forwardable
 
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
                 body doctype content=
-                headers cookies network downloads
+                headers cookies network accessibility downloads
                 mouse keyboard
                 screenshot pdf mhtml viewport_size device_pixel_ratio
                 start_screencast stop_screencast
@@ -34,7 +44,8 @@ module Ferrum
 
     attr_reader :client, :process, :contexts, :options
 
-    delegate %i[timeout timeout= base_url base_url= default_user_agent default_user_agent= extensions] => :options
+    delegate %i[timeout timeout= protocol_timeout protocol_timeout=
+                base_url base_url= default_user_agent default_user_agent= extensions] => :options
     delegate %i[command] => :client
 
     #
@@ -70,18 +81,23 @@ module Ferrum
     #   When present, debug output is written to this object.
     #
     # @option options [Integer, Float] :slowmo
-    #   Set a delay in seconds to wait before sending command.
-    #   Useful companion of headless option, so that you have time to see
+    #   Set a delay in seconds to wait before sending a command.
+    #   Useful companion of a headless option, so that you have time to see
     #   changes.
     #
     # @option options [Numeric] :timeout (5)
-    #   The number of seconds we'll wait for a response when communicating with
-    #   browser.
+    #   The number of seconds we'll wait for a response when communicating
+    #   with the browser: navigations, JS evaluation, DOM queries, dispatching input, etc.
+    #
+    # @option options [Numeric] :protocol_timeout (5)
+    #   The number of seconds we'll wait for an individual internal CDP
+    #   bookkeeping call to respond, e.g. `Target.createTarget`,
+    #   `Target.attachToTarget`. These normally resolve in milliseconds.
     #
     # @option options [Boolean] :js_errors
     #   When true, JavaScript errors get re-raised in Ruby.
     #
-    # @option options [Boolean] :pending_connection_errors (false)
+    # @option options [Boolean] :pending_connection_errors (true)
     #   When main frame is still waiting for slow responses while timeout is
     #   reached {PendingConnectionsError} is raised. It's better to figure out
     #   why you have slow responses and fix or block them rather than turn this
@@ -172,7 +188,7 @@ module Ferrum
     ensure
       if block_given?
         page&.close
-        context.dispose if new_context
+        context&.dispose if new_context
       end
     end
 
@@ -211,11 +227,17 @@ module Ferrum
       contexts.reset
     end
 
+    #
+    # Restarts the browser process, keeping the same options.
+    #
     def restart
       quit
       start
     end
 
+    #
+    # Terminates the browser process and closes the client connection.
+    #
     def quit
       return unless @client
 

data/lib/ferrum/client/subscriber.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Client
+    #
+    # Dispatches incoming CDP events to registered callbacks. Messages are
+    # queued and processed on dedicated threads, with `Fetch.requestPaused`
+    # and `Fetch.authRequired` given priority so request interception isn't
+    # delayed behind other events.
+    #
     class Subscriber
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
@@ -13,6 +19,16 @@ module Ferrum
         start
       end
 
+      #
+      # Enqueues an incoming CDP message for dispatch to subscribers.
+      # `Fetch.requestPaused`/`Fetch.authRequired` messages jump the regular
+      # queue so request interception isn't delayed behind other events.
+      #
+      # @param [Hash] message
+      #   The raw CDP message, as parsed from the websocket.
+      #
+      # @return [void]
+      #
       def <<(message)
         if INTERRUPTIONS.include?(message["method"])
           @priority.push(message)
@@ -21,26 +37,68 @@ module Ferrum
         end
       end
 
+      #
+      # Registers a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Integer]
+      #   The callback's index within the event's callback list, used to
+      #   unsubscribe via {#off}.
+      #
       def on(event, &block)
         @on[event] ||= Concurrent::Array.new
         @on[event] << block
         @on[event].index(block)
       end
 
+      #
+      # Unregisters a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @param [Integer] id
+      #   The callback's index, as returned by {#on}.
+      #
+      # @return [Boolean]
+      #
       def off(event, id)
         @on[event].delete_at(id)
         true
       end
 
+      #
+      # Whether there's at least one callback registered for the event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Boolean]
+      #
       def subscribed?(event)
         @on.key?(event)
       end
 
+      #
+      # Stops the regular and priority dispatch threads.
+      #
+      # @return [void]
+      #
       def close
         @regular_thread&.kill
         @priority_thread&.kill
       end
 
+      #
+      # Removes all callbacks registered for a given session.
+      #
+      # @param [String] session_id
+      #   The session id to match against registered event keys.
+      #
+      # @return [void]
+      #
       def clear(session_id:)
         @on.delete_if { |k, _| k.match?(session_id) }
       end

data/lib/ferrum/client/web_socket.rb

@@ -6,6 +6,12 @@ require "websocket/driver"
 
 module Ferrum
   class Client
+    #
+    # Low-level WebSocket connection to the browser's CDP endpoint. Opens
+    # the raw TCP/TLS socket, drives the `websocket-driver` handshake and
+    # framing, and exposes a queue of parsed incoming messages alongside
+    # methods to send commands and close the connection.
+    #
     class WebSocket
       WEBSOCKET_BUG_SLEEP = 0.05
       DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
@@ -30,7 +36,10 @@ module Ferrum
         end
 
         max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
-        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        @driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        # websocket-driver holds no locks and is called from many threads: commands from
+        # callers, pong/close replies from the reader. One lock keeps frames from interleaving.
+        @driver_mutex = Mutex.new
         @messages = Queue.new
 
         @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
@@ -41,22 +50,28 @@ module Ferrum
 
         start
 
-        @driver.start
+        @driver_mutex.synchronize { @driver.start }
       end
 
+      #
+      # Handles the driver's `:open` event.
+      #
+      # @return [void]
+      #
       def on_open(_event)
         # https://github.com/faye/websocket-driver-ruby/issues/46
         sleep(WEBSOCKET_BUG_SLEEP)
       end
 
+      #
+      # Handles the driver's `:message` event: parses the incoming frame
+      # as JSON and pushes it onto {#messages}. Malformed payloads are
+      # dropped rather than raised, to avoid crashing the reader thread.
+      #
+      # @return [void]
+      #
       def on_message(event)
         data = safely_parse_json(event.data)
-        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
-        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
-        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
-        # thread and the application.
-        @messages.push(data) if data
-
         output = event.data
         if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
           @screenshot_commands.delete(data&.dig("id"))
@@ -64,30 +79,66 @@ module Ferrum
         end
 
         @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
+
+        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
+        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
+        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
+        # thread and the application.
+        @messages.push(data) if data
       end
 
+      #
+      # Handles the driver's `:close` event: closes the message queue and
+      # underlying socket, then kills the reader thread.
+      #
+      # @return [void]
+      #
       def on_close(_event)
         @messages.close
         @sock.close
         @thread.kill
       end
 
+      #
+      # Serializes a CDP command to JSON and sends it as a websocket text
+      # frame.
+      #
+      # @param [Hash] data
+      #   The message to send, must include an `:id` key.
+      #
+      # @return [void]
+      #
       def send_message(data)
         @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
-        @driver.text(json)
+        @driver_mutex.synchronize { @driver.text(json) }
         @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
+      #
+      # Writes raw bytes to the underlying socket. Called by
+      # `websocket-driver` to emit frames. Closes {#messages} instead of
+      # raising if the connection has already been torn down.
+      #
+      # @param [String] data
+      #   The raw bytes to write.
+      #
+      # @return [void]
+      #
       def write(data)
         @sock.write(data)
       rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
         @messages.close
       end
 
+      #
+      # Closes the websocket connection by sending a close frame.
+      #
+      # @return [void]
+      #
       def close
-        @driver.close
+        @driver_mutex.synchronize { @driver.close }
       end
 
       private
@@ -98,7 +149,7 @@ module Ferrum
             data = @sock.readpartial(512)
             break unless data
 
-            @driver.parse(data)
+            @driver_mutex.synchronize { @driver.parse(data) }
           end
         rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
           @messages.close