ferrum 0.17.2 → 0.19.0

data/lib/ferrum/client/subscriber.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Client
+    #
+    # Dispatches incoming CDP events to registered callbacks. Messages are
+    # queued and processed on dedicated threads, with `Fetch.requestPaused`
+    # and `Fetch.authRequired` given priority so request interception isn't
+    # delayed behind other events.
+    #
     class Subscriber
       INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
 
@@ -13,6 +19,16 @@ module Ferrum
         start
       end
 
+      #
+      # Enqueues an incoming CDP message for dispatch to subscribers.
+      # `Fetch.requestPaused`/`Fetch.authRequired` messages jump the regular
+      # queue so request interception isn't delayed behind other events.
+      #
+      # @param [Hash] message
+      #   The raw CDP message, as parsed from the websocket.
+      #
+      # @return [void]
+      #
       def <<(message)
         if INTERRUPTIONS.include?(message["method"])
           @priority.push(message)
@@ -21,26 +37,68 @@ module Ferrum
         end
       end
 
+      #
+      # Registers a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Integer]
+      #   The callback's index within the event's callback list, used to
+      #   unsubscribe via {#off}.
+      #
       def on(event, &block)
         @on[event] ||= Concurrent::Array.new
         @on[event] << block
         @on[event].index(block)
       end
 
+      #
+      # Unregisters a callback for a CDP event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @param [Integer] id
+      #   The callback's index, as returned by {#on}.
+      #
+      # @return [Boolean]
+      #
       def off(event, id)
         @on[event].delete_at(id)
         true
       end
 
+      #
+      # Whether there's at least one callback registered for the event.
+      #
+      # @param [String] event
+      #   The event key, as built by {SessionClient.event_name}.
+      #
+      # @return [Boolean]
+      #
       def subscribed?(event)
         @on.key?(event)
       end
 
+      #
+      # Stops the regular and priority dispatch threads.
+      #
+      # @return [void]
+      #
       def close
         @regular_thread&.kill
         @priority_thread&.kill
       end
 
+      #
+      # Removes all callbacks registered for a given session.
+      #
+      # @param [String] session_id
+      #   The session id to match against registered event keys.
+      #
+      # @return [void]
+      #
       def clear(session_id:)
         @on.delete_if { |k, _| k.match?(session_id) }
       end
@@ -48,7 +106,7 @@ module Ferrum
       private
 
       def start
-        @regular_thread = Utils::Thread.spawn(abort_on_exception: false) do
+        @regular_thread = Utils::Thread.spawn do
           loop do
             message = @regular.pop
             break unless message
@@ -57,7 +115,7 @@ module Ferrum
           end
         end
 
-        @priority_thread = Utils::Thread.spawn(abort_on_exception: false) do
+        @priority_thread = Utils::Thread.spawn do
           loop do
             message = @priority.pop
             break unless message
@@ -75,6 +133,11 @@ module Ferrum
         @on[event]&.each_with_index do |block, index|
           # In case of multiple callbacks we provide current index and total
           block.call(params, index, total)
+        rescue StandardError => e
+          # A raising callback used to terminate the dispatch thread, and with it every
+          # event for the rest of the browser's life: targets stopped being registered
+          # and any page created later raised `NoSuchTargetError`. Report and carry on.
+          warn("Ferrum: #{event} callback raised #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
         end
       end
     end

data/lib/ferrum/client/web_socket.rb

@@ -6,6 +6,12 @@ require "websocket/driver"
 
 module Ferrum
   class Client
+    #
+    # Low-level WebSocket connection to the browser's CDP endpoint. Opens
+    # the raw TCP/TLS socket, drives the `websocket-driver` handshake and
+    # framing, and exposes a queue of parsed incoming messages alongside
+    # methods to send commands and close the connection.
+    #
     class WebSocket
       WEBSOCKET_BUG_SLEEP = 0.05
       DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
@@ -30,7 +36,10 @@ module Ferrum
         end
 
         max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
-        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        @driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
+        # websocket-driver holds no locks and is called from many threads: commands from
+        # callers, pong/close replies from the reader. One lock keeps frames from interleaving.
+        @driver_mutex = Mutex.new
         @messages = Queue.new
 
         @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
@@ -41,22 +50,28 @@ module Ferrum
 
         start
 
-        @driver.start
+        @driver_mutex.synchronize { @driver.start }
       end
 
+      #
+      # Handles the driver's `:open` event.
+      #
+      # @return [void]
+      #
       def on_open(_event)
         # https://github.com/faye/websocket-driver-ruby/issues/46
         sleep(WEBSOCKET_BUG_SLEEP)
       end
 
+      #
+      # Handles the driver's `:message` event: parses the incoming frame
+      # as JSON and pushes it onto {#messages}. Malformed payloads are
+      # dropped rather than raised, to avoid crashing the reader thread.
+      #
+      # @return [void]
+      #
       def on_message(event)
         data = safely_parse_json(event.data)
-        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
-        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
-        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
-        # thread and the application.
-        @messages.push(data) if data
-
         output = event.data
         if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
           @screenshot_commands.delete(data&.dig("id"))
@@ -64,43 +79,141 @@ module Ferrum
         end
 
         @logger&.puts("    ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
+
+        # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
+        # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
+        # anything or skip the background notification, but at least we don't crash the thread that crashes the main
+        # thread and the application.
+        @messages.push(data) if data
       end
 
+      #
+      # Handles the driver's `:close` event: closes the message queue and
+      # underlying socket, then kills the reader thread.
+      #
+      # @return [void]
+      #
       def on_close(_event)
         @messages.close
         @sock.close
         @thread.kill
       end
 
-      def send_message(data)
+      #
+      # Serializes a CDP command to JSON and sends it as a websocket text
+      # frame.
+      #
+      # @param [Hash] data
+      #   The message to send, must include an `:id` key.
+      #
+      # @param [Numeric, nil] timeout
+      #   How long the browser may go without accepting any of the frame's
+      #   bytes. `nil` waits for as long as it takes.
+      #
+      # @return [void]
+      #
+      # @raise [Ferrum::DeadBrowserError]
+      #   If the browser accepted no data for `timeout` seconds. The connection
+      #   is closed, since the frame was cut off midway.
+      #
+      def send_message(data, timeout = nil)
         @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
-        @driver.text(json)
+        @driver_mutex.synchronize do
+          @write_timeout = timeout
+          @driver.text(json)
+        ensure
+          @write_timeout = nil
+        end
         @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
+      #
+      # Writes raw bytes to the underlying socket. Called by
+      # `websocket-driver` to emit frames. Closes {#messages} instead of
+      # raising if the connection has already been torn down.
+      #
+      # A browser that stops reading its socket (hung, or stopped) would block
+      # a plain write forever, holding the lock every other command needs to be
+      # sent. So the write gives up once the browser accepts no data for the
+      # timeout of the command being sent.
+      #
+      # @param [String] data
+      #   The raw bytes to write.
+      #
+      # @return [void]
+      #
+      # @raise [Ferrum::DeadBrowserError]
+      #   If the browser accepted no data for the command's timeout.
+      #
       def write(data)
-        @sock.write(data)
+        until data.empty?
+          written = @sock.write_nonblock(data, exception: false)
+          if written.is_a?(Integer)
+            data = data.byteslice(written..)
+          elsif !wait_for_socket(written)
+            abandon_stalled_write
+          end
+        end
       rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
         @messages.close
       end
 
+      #
+      # Closes the websocket connection by sending a close frame.
+      #
+      # @return [void]
+      #
       def close
-        @driver.close
+        @driver_mutex.synchronize { @driver.close }
+      end
+
+      #
+      # Closes the underlying socket whether or not the close handshake ever
+      # completed.
+      #
+      # {#close} only sends a close frame; the socket itself is closed by
+      # {#on_close}, which runs when the browser answers with a close frame of
+      # its own. A browser that is already dead never answers, so without this
+      # the socket stays open until GC finalizes it -- long enough to exhaust
+      # the process's file descriptors when browsers are restarted in a loop.
+      #
+      # @return [void]
+      #
+      def force_close
+        @sock.close unless @sock.closed?
+      rescue IOError, SystemCallError
+        # nop
       end
 
       private
 
+      def wait_for_socket(state)
+        readers, writers = state == :wait_readable ? [[@sock], nil] : [nil, [@sock]]
+        IO.select(readers, writers, nil, @write_timeout)
+      end
+
+      def abandon_stalled_write
+        timeout = @write_timeout
+        @messages.close
+        @sock.close
+        raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
+      end
+
       def start
         @thread = Utils::Thread.spawn do
           loop do
             data = @sock.readpartial(512)
             break unless data
 
-            @driver.parse(data)
+            @driver_mutex.synchronize { @driver.parse(data) }
           end
         rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
+          # The browser went away, nothing to report.
+        rescue StandardError => e
+          warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
+        ensure
           @messages.close
         end
       end

data/lib/ferrum/client.rb

@@ -7,9 +7,29 @@ require "ferrum/client/web_socket"
 require "ferrum/utils/thread"
 
 module Ferrum
+  #
+  # A thin wrapper around {Client} that scopes commands and events to a
+  # single CDP session (a `sessionId` obtained via `Target.attachToTarget`).
+  # Targets connect through one of these rather than the top-level {Client}
+  # directly, so their commands/events don't leak into other sessions.
+  # Method calls not defined here are forwarded to the underlying {Client}.
+  #
   class SessionClient
     attr_reader :client, :session_id
 
+    #
+    # Builds the internal event key used to route an event to a specific
+    # session, joining the CDP event name with a session id (or leaving it
+    # session-less when `session_id` is `nil`).
+    #
+    # @param [String] event
+    #   The CDP event name, e.g. `"Page.loadEventFired"`.
+    #
+    # @param [String, nil] session_id
+    #   The target session id, or `nil` for the browser-wide session.
+    #
+    # @return [String]
+    #
     def self.event_name(event, session_id)
       [event, session_id].compact.join("_")
     end
@@ -19,31 +39,93 @@ module Ferrum
       @session_id = session_id
     end
 
-    def command(method, async: false, **params)
+    #
+    # Sends a CDP command scoped to this session.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Page.navigate"`.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for this command's response, overriding
+    #   {Browser::Options#protocol_timeout}. See {Client#send_message}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
+    #
+    def command(method, async: false, timeout: nil, **params)
       message = build_message(method, params)
-      @client.send_message(message, async: async)
+      @client.send_message(message, async: async, timeout: timeout)
     end
 
+    #
+    # Subscribes to a CDP event scoped to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    #
     def on(event, &)
       @client.on(event_name(event), &)
     end
 
+    #
+    # Unsubscribes from a CDP event scoped to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    #
     def off(event, id)
       @client.off(event_name(event), id)
     end
 
+    # Whether there's at least one callback registered for the event, scoped
+    # to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Boolean]
     def subscribed?(event)
       @client.subscribed?(event_name(event))
     end
 
+    # Supports {#method_missing} delegation by reporting the underlying
+    # {Client}'s methods as responded to.
+    #
+    # @return [Boolean]
     def respond_to_missing?(name, include_private)
       @client.respond_to?(name, include_private)
     end
 
+    #
+    # Delegates any method not defined on `SessionClient` to the underlying
+    # {Client}, e.g. `subscribed?`.
+    #
+    # @return [untyped]
+    #
     def method_missing(name, ...)
       @client.send(name, ...)
     end
 
+    #
+    # Removes all event callbacks registered for this session from the
+    # underlying client's subscriber.
+    #
+    # @return [void]
+    #
     def close
       @client.subscriber.clear(session_id: session_id)
     end
@@ -59,15 +141,22 @@ module Ferrum
     end
   end
 
+  #
+  # The low-level CDP client. Owns the {WebSocket} connection to the browser,
+  # assigns command ids, matches responses back to their pending commands and
+  # dispatches incoming events to the {Subscriber}. {SessionClient} builds on
+  # top of it to scope commands/events to a particular target's session.
+  #
   class Client
     extend Forwardable
 
-    delegate %i[timeout timeout=] => :options
+    delegate %i[protocol_timeout protocol_timeout=] => :options
 
     attr_reader :ws_url, :options, :subscriber
 
     def initialize(ws_url, options)
       @command_id = 0
+      @command_id_mutex = Mutex.new
       @ws_url = ws_url
       @options = options
       @pendings = Concurrent::Hash.new
@@ -77,24 +166,75 @@ module Ferrum
       start
     end
 
-    def command(method, async: false, **params)
+    #
+    # Sends a CDP command to the browser-wide session.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Target.createTarget"`.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for this command's response, overriding
+    #   {Browser::Options#protocol_timeout}. See {#send_message}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
+    #
+    def command(method, async: false, timeout: nil, **params)
       message = build_message(method, params)
-      send_message(message, async: async)
+      send_message(message, async: async, timeout: timeout)
     end
 
-    def send_message(message, async:)
+    #
+    # Sends a raw CDP message over the websocket. Synchronous calls block
+    # until a matching response arrives, or `timeout` elapses, defaulting to
+    # `protocol_timeout` (delegated to {Browser::Options#protocol_timeout}).
+    # That default is the transport-level budget for internal CDP bookkeeping
+    # (e.g. `Target.createTarget`). {Page#command} overrides
+    # this back to `timeout`, or a caller-supplied budget (e.g. `#pdf`/
+    # `#screenshot`'s own `timeout:` argument), for the user-facing commands
+    # it issues -- some of which (e.g. `Page.navigate`, `Page.printToPDF`)
+    # rely on their own response latency to detect a stuck operation.
+    #
+    # @param [Hash] message
+    #   The message to send, must include an `:id` key.
+    #
+    # @param [Boolean] async
+    #   Whether to return immediately instead of waiting for a response.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for the browser to take the message, and then for its
+    #   response. Defaults to `protocol_timeout`.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the parsed `"result"`
+    #   from the response.
+    #
+    def send_message(message, async:, timeout: nil)
+      timeout ||= protocol_timeout
+
       if async
-        @ws.send_message(message)
+        @ws.send_message(message, timeout)
         true
       else
+        raise DeadBrowserError if @ws.messages.closed?
+
         pending = Concurrent::IVar.new
         @pendings[message[:id]] = pending
-        @ws.send_message(message)
-        data = pending.value!(timeout)
-        @pendings.delete(message[:id])
+        begin
+          @ws.send_message(message, timeout)
+          data = pending.value!(timeout)
+        ensure
+          @pendings.delete(message[:id])
+        end
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
-        raise TimeoutError unless data
+        raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
 
         error, response = data.values_at("error", "result")
         raise_browser_error(error) if error
@@ -102,30 +242,77 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to a CDP event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    #
     def on(event, &)
       @subscriber.on(event, &)
     end
 
+    #
+    # Unsubscribes from a CDP event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    #
     def off(event, id)
       @subscriber.off(event, id)
     end
 
+    # Whether there's at least one callback registered for the event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Boolean]
     def subscribed?(event)
       @subscriber.subscribed?(event)
     end
 
+    #
+    # Builds a client scoped to a given CDP session, e.g. a browsing
+    # context created via `Target.attachToTarget`.
+    #
+    # @param [String] session_id
+    #   The CDP session id to scope commands and events to.
+    #
+    # @return [SessionClient]
+    #
     def session(session_id)
       SessionClient.new(self, session_id)
     end
 
+    #
+    # Closes the underlying websocket, drops pending commands and stops
+    # the message-processing thread and subscriber.
+    #
+    # @return [void]
+    #
     def close
       @ws.close
       # Give a thread some time to handle a tail of messages
       @pendings.clear
       @thread.kill unless @thread.join(1)
       @subscriber.close
+      @ws.force_close
     end
 
+    #
+    # Custom inspection that exposes internal state useful for debugging.
+    #
+    # @return [String]
+    #
     def inspect
       "#<#{self.class} " \
         "@command_id=#{@command_id.inspect} " \
@@ -133,6 +320,17 @@ module Ferrum
         "@ws=#{@ws.inspect}>"
     end
 
+    #
+    # Builds a CDP message hash with a fresh, thread-safe command id.
+    #
+    # @param [String] method
+    #   The CDP method name.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @return [Hash]
+    #
     def build_message(method, params)
       { method: method, params: params }.merge(id: next_command_id)
     end
@@ -151,11 +349,26 @@ module Ferrum
             @pendings[message["id"]]&.set(message)
           end
         end
+      rescue StandardError => e
+        # Nothing delivers responses once this thread is gone, so the connection
+        # is done for. Close it rather than leave every command hanging.
+        warn("Ferrum: dispatch stopped, #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
+        @ws.messages.close
+      ensure
+        release_pendings
       end
     end
 
+    # Nothing is going to answer the commands still in flight once the socket
+    # is gone, release them instead of making each wait out its timeout.
+    def release_pendings
+      @pendings.each_value { |pending| pending.try_set(nil) }
+    end
+
+    # Locked so two concurrent commands never share an id and read each
+    # other's responses from @pendings.
     def next_command_id
-      @command_id += 1
+      @command_id_mutex.synchronize { @command_id += 1 }
     end
 
     def raise_browser_error(error)