ferrum 0.17.1 → 0.18.0

data/lib/ferrum/client.rb

@@ -7,9 +7,29 @@ require "ferrum/client/web_socket"
 require "ferrum/utils/thread"
 
 module Ferrum
+  #
+  # A thin wrapper around {Client} that scopes commands and events to a
+  # single CDP session (a `sessionId` obtained via `Target.attachToTarget`).
+  # Targets connect through one of these rather than the top-level {Client}
+  # directly, so their commands/events don't leak into other sessions.
+  # Method calls not defined here are forwarded to the underlying {Client}.
+  #
   class SessionClient
     attr_reader :client, :session_id
 
+    #
+    # Builds the internal event key used to route an event to a specific
+    # session, joining the CDP event name with a session id (or leaving it
+    # session-less when `session_id` is `nil`).
+    #
+    # @param [String] event
+    #   The CDP event name, e.g. `"Page.loadEventFired"`.
+    #
+    # @param [String, nil] session_id
+    #   The target session id, or `nil` for the browser-wide session.
+    #
+    # @return [String]
+    #
     def self.event_name(event, session_id)
       [event, session_id].compact.join("_")
     end
@@ -19,31 +39,93 @@ module Ferrum
       @session_id = session_id
     end
 
-    def command(method, async: false, **params)
+    #
+    # Sends a CDP command scoped to this session.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Page.navigate"`.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for this command's response, overriding
+    #   {Browser::Options#protocol_timeout}. See {Client#send_message}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
+    #
+    def command(method, async: false, timeout: nil, **params)
       message = build_message(method, params)
-      @client.send_message(message, async: async)
+      @client.send_message(message, async: async, timeout: timeout)
     end
 
-    def on(event, &block)
-      @client.on(event_name(event), &block)
+    #
+    # Subscribes to a CDP event scoped to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    #
+    def on(event, &)
+      @client.on(event_name(event), &)
     end
 
+    #
+    # Unsubscribes from a CDP event scoped to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    #
     def off(event, id)
       @client.off(event_name(event), id)
     end
 
+    # Whether there's at least one callback registered for the event, scoped
+    # to this session.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Boolean]
     def subscribed?(event)
       @client.subscribed?(event_name(event))
     end
 
+    # Supports {#method_missing} delegation by reporting the underlying
+    # {Client}'s methods as responded to.
+    #
+    # @return [Boolean]
     def respond_to_missing?(name, include_private)
       @client.respond_to?(name, include_private)
     end
 
-    def method_missing(name, *args, **opts, &block)
-      @client.send(name, *args, **opts, &block)
+    #
+    # Delegates any method not defined on `SessionClient` to the underlying
+    # {Client}, e.g. `subscribed?`.
+    #
+    # @return [untyped]
+    #
+    def method_missing(name, ...)
+      @client.send(name, ...)
     end
 
+    #
+    # Removes all event callbacks registered for this session from the
+    # underlying client's subscriber.
+    #
+    # @return [void]
+    #
     def close
       @client.subscriber.clear(session_id: session_id)
     end
@@ -59,14 +141,22 @@ module Ferrum
     end
   end
 
+  #
+  # The low-level CDP client. Owns the {WebSocket} connection to the browser,
+  # assigns command ids, matches responses back to their pending commands and
+  # dispatches incoming events to the {Subscriber}. {SessionClient} builds on
+  # top of it to scope commands/events to a particular target's session.
+  #
   class Client
     extend Forwardable
-    delegate %i[timeout timeout=] => :options
+
+    delegate %i[protocol_timeout protocol_timeout=] => :options
 
     attr_reader :ws_url, :options, :subscriber
 
     def initialize(ws_url, options)
       @command_id = 0
+      @command_id_mutex = Mutex.new
       @ws_url = ws_url
       @options = options
       @pendings = Concurrent::Hash.new
@@ -76,12 +166,55 @@ module Ferrum
       start
     end
 
-    def command(method, async: false, **params)
+    #
+    # Sends a CDP command to the browser-wide session.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Target.createTarget"`.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for this command's response, overriding
+    #   {Browser::Options#protocol_timeout}. See {#send_message}.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the command's result.
+    #
+    def command(method, async: false, timeout: nil, **params)
       message = build_message(method, params)
-      send_message(message, async: async)
+      send_message(message, async: async, timeout: timeout)
     end
 
-    def send_message(message, async:)
+    #
+    # Sends a raw CDP message over the websocket. Synchronous calls block
+    # until a matching response arrives, or `timeout` elapses, defaulting to
+    # `protocol_timeout` (delegated to {Browser::Options#protocol_timeout}).
+    # That default is the transport-level budget for internal CDP bookkeeping
+    # (e.g. `Target.createTarget`). {Page#command} overrides
+    # this back to `timeout`, or a caller-supplied budget (e.g. `#pdf`/
+    # `#screenshot`'s own `timeout:` argument), for the user-facing commands
+    # it issues -- some of which (e.g. `Page.navigate`, `Page.printToPDF`)
+    # rely on their own response latency to detect a stuck operation.
+    #
+    # @param [Hash] message
+    #   The message to send, must include an `:id` key.
+    #
+    # @param [Boolean] async
+    #   Whether to return immediately instead of waiting for a response.
+    #
+    # @param [Numeric, nil] timeout
+    #   How long to wait for the response. Defaults to `protocol_timeout`.
+    #
+    # @return [Boolean, Hash]
+    #   `true` when sent asynchronously, otherwise the parsed `"result"`
+    #   from the response.
+    #
+    def send_message(message, async:, timeout: nil)
       if async
         @ws.send_message(message)
         true
@@ -89,7 +222,7 @@ module Ferrum
         pending = Concurrent::IVar.new
         @pendings[message[:id]] = pending
         @ws.send_message(message)
-        data = pending.value!(timeout)
+        data = pending.value!(timeout || protocol_timeout)
         @pendings.delete(message[:id])
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
@@ -101,22 +234,63 @@ module Ferrum
       end
     end
 
-    def on(event, &block)
-      @subscriber.on(event, &block)
+    #
+    # Subscribes to a CDP event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
+    #
+    def on(event, &)
+      @subscriber.on(event, &)
     end
 
+    #
+    # Unsubscribes from a CDP event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
+    #
     def off(event, id)
       @subscriber.off(event, id)
     end
 
+    # Whether there's at least one callback registered for the event.
+    #
+    # @param [String] event
+    #   The CDP event name.
+    #
+    # @return [Boolean]
     def subscribed?(event)
       @subscriber.subscribed?(event)
     end
 
+    #
+    # Builds a client scoped to a given CDP session, e.g. a browsing
+    # context created via `Target.attachToTarget`.
+    #
+    # @param [String] session_id
+    #   The CDP session id to scope commands and events to.
+    #
+    # @return [SessionClient]
+    #
     def session(session_id)
       SessionClient.new(self, session_id)
     end
 
+    #
+    # Closes the underlying websocket, drops pending commands and stops
+    # the message-processing thread and subscriber.
+    #
+    # @return [void]
+    #
     def close
       @ws.close
       # Give a thread some time to handle a tail of messages
@@ -125,6 +299,11 @@ module Ferrum
       @subscriber.close
     end
 
+    #
+    # Custom inspection that exposes internal state useful for debugging.
+    #
+    # @return [String]
+    #
     def inspect
       "#<#{self.class} " \
         "@command_id=#{@command_id.inspect} " \
@@ -132,6 +311,17 @@ module Ferrum
         "@ws=#{@ws.inspect}>"
     end
 
+    #
+    # Builds a CDP message hash with a fresh, thread-safe command id.
+    #
+    # @param [String] method
+    #   The CDP method name.
+    #
+    # @param [Hash] params
+    #   The command's parameters.
+    #
+    # @return [Hash]
+    #
     def build_message(method, params)
       { method: method, params: params }.merge(id: next_command_id)
     end
@@ -153,8 +343,10 @@ module Ferrum
       end
     end
 
+    # Locked so two concurrent commands never share an id and read each
+    # other's responses from @pendings.
     def next_command_id
-      @command_id += 1
+      @command_id_mutex.synchronize { @command_id += 1 }
     end
 
     def raise_browser_error(error)

data/lib/ferrum/context.rb

@@ -3,6 +3,13 @@
 require "ferrum/target"
 
 module Ferrum
+  #
+  # Represents a browser context, i.e. an isolated browsing profile (similar
+  # to an incognito window) with its own cookies, cache and storage. Keeps
+  # track of the {Target}s that belong to it and connects to them as {Page}s
+  # or {Worker}s. Managed by {Contexts}, which owns the browser's collection
+  # of contexts and routes CDP target events to the right one.
+  #
   class Context
     POSITION = %i[first last].freeze
 
@@ -16,16 +23,43 @@ module Ferrum
       @pendings = Concurrent::Map.new
     end
 
+    # The context's first known target, creating one via
+    # `Target.createTarget` if none has attached yet.
+    #
+    # @return [Target]
     def default_target
       @default_target ||= create_target
     end
 
+    # Connects to and returns the {#default_target}'s page.
+    #
+    # @return [Page]
     def page
       default_target.page
     end
 
+    # All page targets in this context, connected to as {Page}s.
+    #
+    # @return [Array<Page>]
     def pages
-      @targets.values.reject(&:iframe?).map(&:page)
+      @targets.values.select(&:page?).map(&:page)
+    end
+
+    # Dedicated and shared workers spawned by any page in this context.
+    #
+    # @return [Array<Worker>]
+    def workers
+      @targets.values.select { |t| t.worker? || t.shared_worker? }.map(&:worker)
+    end
+
+    # Service worker targets registered in this context. Unlike {#workers},
+    # these are plain {Target}s and are not connected to. Attaching to a
+    # service worker's session keeps it alive indefinitely, so we only do
+    # that on demand, via `target.worker`.
+    #
+    # @return [Array<Target>]
+    def service_workers
+      @targets.values.select(&:service_worker?)
     end
 
     # When we call `page` method on target it triggers ruby to connect to given
@@ -40,74 +74,135 @@ module Ferrum
       windows.map(&:page)
     end
 
+    # Creates a new target in this context and eagerly connects to it as a
+    # {Page}, forwarding `options` to {Target#build_page}.
+    #
+    # @param [Hash] options
+    #
+    # @return [Page]
     def create_page(**options)
       target = create_target
       target.page = target.build_page(**options)
     end
 
+    # Creates a new target in this context via `Target.createTarget` and
+    # blocks until it's been registered (see {#add_target}).
+    #
+    # @return [Target]
+    #
+    # @raise [NoSuchTargetError]
     def create_target
       target_id = @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")["targetId"]
 
       new_pending = Concurrent::IVar.new
       pending = @pendings.put_if_absent(target_id, new_pending) || new_pending
-      resolved = pending.value(@client.timeout)
+      resolved = pending.value(@client.protocol_timeout)
       raise NoSuchTargetError unless resolved
 
       @pendings.delete(target_id)
       @targets[target_id]
     end
 
+    # Registers a target discovered via a CDP `Target.*` event, or updates
+    # the session id on one already known. Called by {Contexts} as targets
+    # are created/attached.
+    #
+    # @param [Hash] params
+    #   The target's `targetInfo`.
+    #
+    # @param [String, nil] session_id
+    #
+    # @return [Target]
     def add_target(params:, session_id: nil)
       new_target = Target.new(@client, session_id, params)
       # `put_if_absent` returns nil if added a new value or existing if there was one already
       target = @targets.put_if_absent(new_target.id, new_target) || new_target
+      # on first iteration session_id may be nil, then if session is present here we must set it to the target
+      target.session_id = session_id if session_id && target.session_id.nil?
       @default_target ||= target
 
       new_pending = Concurrent::IVar.new
       pending = @pendings.put_if_absent(target.id, new_pending) || new_pending
       pending.try_set(true)
-      true
+      target
     end
 
+    # Updates a known target's params, e.g. on `Target.targetInfoChanged`.
+    #
+    # @param [String] target_id
+    #
+    # @param [Hash] params
+    #
+    # @return [void]
     def update_target(target_id, params)
       @targets[target_id]&.update(params)
     end
 
+    # Removes a target, e.g. on `Target.targetDestroyed`/`targetCrashed`.
+    #
+    # @param [String] target_id
+    #
+    # @return [Target, nil]
     def delete_target(target_id)
       @targets.delete(target_id)
     end
 
+    # Manually attaches to a target, e.g. a service worker discovered via
+    # {#service_workers}. Once attached, `target.worker`/`target.page`
+    # returns a connected {Worker}/{Page} for it.
+    #
+    # Note: attaching to a service worker's session prevents Chrome from
+    # ever terminating it while the connection is open.
     def attach_target(target_id)
       target = @targets[target_id]
       raise NoSuchTargetError unless target
 
+      @contexts.manually_attached(target_id)
       session = @client.command("Target.attachToTarget", targetId: target_id, flatten: true)
       target.session_id = session["sessionId"]
       true
     end
 
+    # Returns the first target for which the block returns truthy.
+    #
+    # @return [Target, nil]
     def find_target
       @targets.each_value { |t| return t if yield(t) }
 
       nil
     end
 
+    # Closes the WebSocket connection of every connected target, without
+    # disposing the targets themselves.
+    #
+    # @return [void]
     def close_targets_connection
       @targets.each_value do |target|
         next unless target.connected?
 
-        target.page.close_connection
+        target.close_connection
       end
     end
 
+    # Disposes this browser context and all of its targets.
+    #
+    # @return [Boolean]
     def dispose
       @contexts.dispose(@id)
     end
 
+    # Whether a target with the given id is known in this context.
+    #
+    # @param [String] target_id
+    #
+    # @return [Boolean]
     def target?(target_id)
       !!@targets[target_id]
     end
 
+    # Debug representation of the context, including its known targets.
+    #
+    # @return [String]
     def inspect
       %(#<#{self.class} @id=#{@id.inspect} @targets=#{@targets.inspect} @default_target=#{@default_target.inspect}>)
     end