ferrum 0.17.2 → 0.19.0

data/lib/ferrum/context.rb

@@ -3,29 +3,74 @@
 require "ferrum/target"
 
 module Ferrum
+  #
+  # Represents a browser context, i.e. an isolated browsing profile (similar
+  # to an incognito window) with its own cookies, cache and storage. Keeps
+  # track of the {Target}s that belong to it and connects to them as {Page}s
+  # or {Worker}s. Managed by {Contexts}, which owns the browser's collection
+  # of contexts and routes CDP target events to the right one.
+  #
   class Context
     POSITION = %i[first last].freeze
 
     attr_reader :id, :targets
 
-    def initialize(client, contexts, id)
+    def initialize(client, contexts, id, implicit: false)
       @id = id
+      @implicit = implicit
       @client = client
       @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::Map.new
     end
 
+    # Whether this is the browser's implicit context, the one it puts its
+    # startup window in. We didn't create it, so we can't dispose it, and
+    # Chrome doesn't let us address it by id either: targets are created in
+    # it by omitting `browserContextId` altogether.
+    #
+    # @return [Boolean]
+    def implicit?
+      @implicit
+    end
+
+    # The context's first known target, creating one via
+    # `Target.createTarget` if none has attached yet.
+    #
+    # @return [Target]
     def default_target
       @default_target ||= create_target
     end
 
+    # Connects to and returns the {#default_target}'s page.
+    #
+    # @return [Page]
     def page
       default_target.page
     end
 
+    # All page targets in this context, connected to as {Page}s.
+    #
+    # @return [Array<Page>]
     def pages
-      @targets.values.reject(&:iframe?).map(&:page)
+      @targets.values.select(&:page?).map(&:page)
+    end
+
+    # Dedicated and shared workers spawned by any page in this context.
+    #
+    # @return [Array<Worker>]
+    def workers
+      @targets.values.select { |t| t.worker? || t.shared_worker? }.map(&:worker)
+    end
+
+    # Service worker targets registered in this context. Unlike {#workers},
+    # these are plain {Target}s and are not connected to. Attaching to a
+    # service worker's session keeps it alive indefinitely, so we only do
+    # that on demand, via `target.worker`.
+    #
+    # @return [Array<Target>]
+    def service_workers
+      @targets.values.select(&:service_worker?)
     end
 
     # When we call `page` method on target it triggers ruby to connect to given
@@ -40,25 +85,48 @@ module Ferrum
       windows.map(&:page)
     end
 
+    # Creates a new target in this context and eagerly connects to it as a
+    # {Page}, forwarding `options` to {Target#build_page}.
+    #
+    # @param [Hash] options
+    #
+    # @return [Page]
     def create_page(**options)
       target = create_target
       target.page = target.build_page(**options)
     end
 
+    # Creates a new target in this context via `Target.createTarget` and
+    # blocks until it's been registered (see {#add_target}).
+    #
+    # @return [Target]
+    #
+    # @raise [NoSuchTargetError]
     def create_target
-      target_id = @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")["targetId"]
+      options = { browserContextId: @id } unless implicit?
+      target_id = @client.command("Target.createTarget", url: "about:blank", **Hash(options))["targetId"]
 
       new_pending = Concurrent::IVar.new
       pending = @pendings.put_if_absent(target_id, new_pending) || new_pending
-      resolved = pending.value(@client.timeout)
+      resolved = pending.value(@client.protocol_timeout)
       raise NoSuchTargetError unless resolved
 
       @pendings.delete(target_id)
       @targets[target_id]
     end
 
+    # Registers a target discovered via a CDP `Target.*` event, or updates
+    # the session id on one already known. Called by {Contexts} as targets
+    # are created/attached.
+    #
+    # @param [Hash] params
+    #   The target's `targetInfo`.
+    #
+    # @param [String, nil] session_id
+    #
+    # @return [Target]
     def add_target(params:, session_id: nil)
-      new_target = Target.new(@client, session_id, params)
+      new_target = Target.new(@client, session_id, params, implicit: implicit?)
       # `put_if_absent` returns nil if added a new value or existing if there was one already
       target = @targets.put_if_absent(new_target.id, new_target) || new_target
       # on first iteration session_id may be nil, then if session is present here we must set it to the target
@@ -68,48 +136,86 @@ module Ferrum
       new_pending = Concurrent::IVar.new
       pending = @pendings.put_if_absent(target.id, new_pending) || new_pending
       pending.try_set(true)
-      true
+      target
     end
 
+    # Updates a known target's params, e.g. on `Target.targetInfoChanged`.
+    #
+    # @param [String] target_id
+    #
+    # @param [Hash] params
+    #
+    # @return [void]
     def update_target(target_id, params)
       @targets[target_id]&.update(params)
     end
 
+    # Removes a target, e.g. on `Target.targetDestroyed`/`targetCrashed`.
+    #
+    # @param [String] target_id
+    #
+    # @return [Target, nil]
     def delete_target(target_id)
       @targets.delete(target_id)
     end
 
+    # Manually attaches to a target, e.g. a service worker discovered via
+    # {#service_workers}. Once attached, `target.worker`/`target.page`
+    # returns a connected {Worker}/{Page} for it.
+    #
+    # Note: attaching to a service worker's session prevents Chrome from
+    # ever terminating it while the connection is open.
     def attach_target(target_id)
       target = @targets[target_id]
       raise NoSuchTargetError unless target
 
+      @contexts.manually_attached(target_id)
       session = @client.command("Target.attachToTarget", targetId: target_id, flatten: true)
       target.session_id = session["sessionId"]
       true
     end
 
+    # Returns the first target for which the block returns truthy.
+    #
+    # @return [Target, nil]
     def find_target
       @targets.each_value { |t| return t if yield(t) }
 
       nil
     end
 
+    # Closes the WebSocket connection of every connected target, without
+    # disposing the targets themselves.
+    #
+    # @return [void]
     def close_targets_connection
       @targets.each_value do |target|
         next unless target.connected?
 
-        target.page.close_connection
+        target.close_connection
       end
     end
 
+    # Disposes this browser context and all of its targets. The browser's
+    # implicit context cannot be disposed, see {#implicit?}.
+    #
+    # @return [Boolean, nil]
     def dispose
       @contexts.dispose(@id)
     end
 
+    # Whether a target with the given id is known in this context.
+    #
+    # @param [String] target_id
+    #
+    # @return [Boolean]
     def target?(target_id)
       !!@targets[target_id]
     end
 
+    # Debug representation of the context, including its known targets.
+    #
+    # @return [String]
     def inspect
       %(#<#{self.class} @id=#{@id.inspect} @targets=#{@targets.inspect} @default_target=#{@default_target.inspect}>)
     end

data/lib/ferrum/contexts.rb

@@ -3,41 +3,82 @@
 require "ferrum/context"
 
 module Ferrum
+  #
+  # Owns the browser's whole collection of {Context}s, keyed by their
+  # `browserContextId`. Subscribes to the CDP `Target.*` events, creates
+  # {Context}s and {Target}s as they're discovered/attached, and routes
+  # target lifecycle updates (info changed, destroyed, crashed) to the
+  # {Context} they belong to.
+  #
   class Contexts
-    ALLOWED_TARGET_TYPES = %w[page iframe].freeze
+    ALLOWED_TARGET_TYPES = %w[page iframe worker shared_worker service_worker].freeze
+    RECURSIVE_AUTO_ATTACH_TYPES = %w[page iframe worker shared_worker].freeze
 
     include Enumerable
 
     attr_reader :contexts
 
     def initialize(client)
-      @contexts = Concurrent::Map.new
       @client = client
+      @contexts = Concurrent::Map.new
+      @manually_attached = Concurrent::Map.new
+      @default_context = find_implicit_context
       subscribe
       auto_attach
       discover
     end
 
+    # Marks a target, so the next time we see it attached, we leave its session alone instead of {#detach}ing it.
+    # Used by {Context#attach_target} right before it manually attaches to a service worker on the caller's behalf.
+    def manually_attached(target_id)
+      @manually_attached[target_id] = true
+    end
+
+    # The context we work in unless told otherwise: the browser's implicit
+    # context when it came up with a startup window (see {#find_implicit_context}),
+    # otherwise one of our own, created lazily.
+    #
+    # @return [Context]
     def default_context
       @default_context ||= create
     end
 
+    # Iterates over `[id, context]` pairs; returns an `Enumerator` if no
+    # block is given.
+    #
+    # @return [void, Enumerator]
     def each(&)
       return enum_for(__method__) unless block_given?
 
       @contexts.each(&)
     end
 
+    # The context with the given id, if any.
+    #
+    # @param [String] id
+    #
+    # @return [Context, nil]
     def [](id)
       @contexts[id]
     end
 
+    # The context that owns the given target, if any.
+    #
+    # @param [String] target_id
+    #
+    # @return [Context, nil]
     def find_by(target_id:)
       context = nil
       @contexts.each_value { |c| context = c if c.target?(target_id) }
       context
     end
 
+    # Creates a new browser context (like an incognito profile).
+    #
+    # @param [Hash] options
+    #   Keyword arguments forwarded to `Target.createBrowserContext`.
+    #
+    # @return [Context]
     def create(**options)
       response = @client.command("Target.createBrowserContext", **options)
       context_id = response["browserContextId"]
@@ -46,46 +87,90 @@ module Ferrum
       context
     end
 
+    # Disposes a browser context and all of its targets. The browser's implicit
+    # context is not ours to dispose, see {Context#implicit?}.
+    #
+    # The context is forgotten even if Chrome fails to dispose it in time: its
+    # targets are disconnected by then, and asking again only gets
+    # `Disposal of browser context ... is already pending` back.
+    #
+    # @param [String] context_id
+    #
+    # @return [Boolean, nil]
+    #
+    # @raise [TimeoutError, BrowserError]
     def dispose(context_id)
       context = @contexts[context_id]
       return unless context
+      return if context.implicit?
 
-      context.close_targets_connection
-      @client.command("Target.disposeBrowserContext", browserContextId: context.id)
-      @contexts.delete(context_id)
+      begin
+        context.close_targets_connection
+        @client.command("Target.disposeBrowserContext", browserContextId: context.id)
+      ensure
+        @contexts.delete(context_id)
+      end
       true
     end
 
+    # Closes the WebSocket connection of every target in every context,
+    # without disposing the contexts themselves.
+    #
+    # @return [void]
     def close_connections
       @contexts.each_value(&:close_targets_connection)
     end
 
+    # Disposes every context still known to the browser. A context that fails
+    # to dispose doesn't stop the others from being disposed; the first error
+    # is raised once all of them have been tried.
+    #
+    # @return [void]
+    #
+    # @raise [TimeoutError, BrowserError]
     def reset
+      error = nil
       context_ids = @client.command("Target.getBrowserContexts")["browserContextIds"]
       @default_context = nil if context_ids.include?(@default_context&.id)
-      @contexts.each_key { |id| dispose(id) if context_ids.include?(id) }
+      (@contexts.keys & context_ids).each do |id|
+        dispose(id)
+      rescue BrowserError, TimeoutError => e
+        error ||= e
+      end
+      raise error if error
     end
 
+    # Number of known contexts.
+    #
+    # @return [Integer]
     def size
       @contexts.size
     end
 
     private
 
-    def subscribe # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
-      @client.on("Target.attachedToTarget") do |params|
+    def subscribe
+      subscribe_attached_target(@client)
+      subscribe_target_created
+    end
+
+    # Registered once on the top-level client, and again on every page's/
+    # worker's own session once we re-arm auto-attach on it.
+    def subscribe_attached_target(client)
+      client.on("Target.attachedToTarget") do |params|
         info, session_id = params.values_at("targetInfo", "sessionId")
-        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
+        next detach_untracked(session_id) unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
         add_context(context_id)
+        target = @contexts[context_id]&.add_target(session_id: session_id, params: info)
 
-        @contexts[context_id]&.add_target(session_id: session_id, params: info)
-        if params["waitingForDebugger"]
-          @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
-        end
+        rearm_auto_attach(session_id, info["type"])
+        handle_attach(target, session_id, params)
       end
+    end
 
+    def subscribe_target_created
       @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
         next unless ALLOWED_TARGET_TYPES.include?(info["type"])
@@ -93,9 +178,8 @@ module Ferrum
         context_id = info["browserContextId"]
         add_context(context_id)
 
-        if info["type"] == "iframe" &&
-           (target = @contexts[context_id]&.find_target { |t| t.connected? && t.page.frame_by(id: info["targetId"]) })
-          @contexts[context_id]&.add_target(session_id: target.page.client.session_id, params: info)
+        if info["type"] == "iframe" && (target = find_frame_owner(context_id, info["targetId"]))
+          @contexts[context_id]&.add_target(session_id: target.session_id, params: info)
         else
           @contexts[context_id]&.add_target(params: info)
         end
@@ -120,6 +204,71 @@ module Ferrum
       end
     end
 
+    def find_frame_owner(context_id, frame_id)
+      @contexts[context_id]&.find_target do |target|
+        (target.page? || target.iframe?) && target.connected? && target.page.frame_by(id: frame_id)
+      end
+    end
+
+    def rearm_auto_attach(session_id, type)
+      return unless RECURSIVE_AUTO_ATTACH_TYPES.include?(type)
+
+      client = @client.session(session_id)
+      client.command("Target.setAutoAttach", autoAttach: true, waitForDebuggerOnStart: true, flatten: true, async: true)
+      subscribe_attached_target(client)
+    end
+
+    def handle_attach(target, session_id, params)
+      return unless target
+
+      if target.service_worker?
+        detach_unless_manually_attached(target, session_id)
+      elsif target.worker? || target.shared_worker?
+        connect_worker(target)
+      elsif params["waitingForDebugger"]
+        resume(session_id)
+      end
+    end
+
+    # Auto-attach pauses every target it attaches to, including the types
+    # we don't track: Chrome's own `browser_ui` targets, for instance. We
+    # never resume those, so left alone they stay attached and paused for
+    # the lifetime of the browser. Resume them and drop the session.
+    def detach_untracked(session_id)
+      detach(session_id)
+    rescue BrowserError, TimeoutError
+      nil
+    end
+
+    # Attaching keeps a service worker alive forever, so unless the caller
+    # explicitly asked to connect to it (via Context#attach_target), we
+    # just resume it and let go.
+    def detach_unless_manually_attached(target, session_id)
+      return if @manually_attached.delete(target.id)
+
+      detach(session_id)
+    rescue BrowserError, TimeoutError
+      nil
+    end
+
+    # Workers have no events to notify us when they're ready, so we
+    # connect right away. Worker#prepare enables the Network domain and
+    # only then resumes the debugger itself.
+    def connect_worker(target)
+      target.worker
+    rescue BrowserError, TimeoutError
+      nil
+    end
+
+    def resume(session_id)
+      @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
+    end
+
+    def detach(session_id)
+      resume(session_id)
+      @client.command("Target.detachFromTarget", sessionId: session_id)
+    end
+
     def discover
       @client.command("Target.setDiscoverTargets", discover: true)
     end
@@ -131,9 +280,29 @@ module Ferrum
     def add_context(context_id)
       return if @contexts[context_id]
 
-      context = Context.new(@client, self, context_id)
-      @contexts[context_id] = context
-      @default_context ||= context # rubocop:disable Naming/MemoizedInstanceVariableName
+      @contexts[context_id] = Context.new(@client, self, context_id)
+    end
+
+    # The browser's implicit context, the context Chrome uses for its startup window.
+    # We register it so events for that window are routed to it.
+    # `Target.getBrowserContexts` reports every context created with
+    # `Target.createBrowserContext`, ours and those of anyone else driving the same
+    # browser, so the context holding targets that it doesn't report is the implicit
+    # one. Chrome also reports its id as `defaultBrowserContextId`, but only since
+    # 145 and only as an experimental field, so we don't rely on it.
+    #
+    # Returns nil if the browser has no startup window, which happens when it is
+    # launched with `--no-startup-window` (used for `incognito: true`). In that case,
+    # we create our own context instead.
+    #
+    # @return [Context, nil]
+    def find_implicit_context
+      created_ids = @client.command("Target.getBrowserContexts")["browserContextIds"]
+      target_infos = @client.command("Target.getTargets")["targetInfos"]
+      implicit_id = (target_infos.filter_map { |t| t["browserContextId"] } - created_ids).first
+      return unless implicit_id
+
+      @contexts[implicit_id] = Context.new(@client, self, implicit_id, implicit: true)
     end
   end
 end

data/lib/ferrum/cookies/cookie.rb

@@ -122,6 +122,8 @@ module Ferrum
         @attributes["priority"]
       end
 
+      #
+      # Specifies whether the cookie is a first-party set cookie or not.
       #
       # @return [Boolean]
       #
@@ -131,6 +133,8 @@ module Ferrum
 
       alias same_party? sameparty?
 
+      #
+      # The scheme (`"Unset"`, `"NonSecure"`, `"Secure"`) that the cookie was set with.
       #
       # @return [String]
       #
@@ -138,6 +142,8 @@ module Ferrum
         @attributes["sourceScheme"]
       end
 
+      #
+      # The port that the cookie was set with, or `-1` if unknown.
       #
       # @return [Integer]
       #

data/lib/ferrum/cookies.rb

@@ -4,6 +4,11 @@ require "yaml"
 require "ferrum/cookies/cookie"
 
 module Ferrum
+  #
+  # Manages a page's cookies via the CDP `Network` domain. Enumerable over
+  # the page's current {Cookies::Cookie}s, and provides methods to read,
+  # set, remove and clear them, as well as persist them to/from a YAML file.
+  #
   class Cookies
     include Enumerable
 

data/lib/ferrum/dialog.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents a JavaScript dialog (`alert`, `confirm`, `prompt` or
+  # `beforeunload`) shown by the page. Instances are yielded to blocks
+  # registered with `page.on(:dialog)`, and can be accepted (optionally with
+  # a prompt's text) or dismissed.
+  #
   class Dialog
     attr_reader :message, :default_prompt
 
@@ -51,8 +57,18 @@ module Ferrum
       @page.command("Page.handleJavaScriptDialog", slowmoable: true, accept: false)
     end
 
-    def match?(regexp)
-      !!message.match(regexp)
+    #
+    # Whether the dialog's message matches the given pattern.
+    #
+    # @param [Regexp, String] pattern
+    #   A regexp matched against the message, or a string checked for inclusion in it.
+    #
+    # @return [Boolean]
+    #
+    def match?(pattern)
+      return message.match?(pattern) if pattern.is_a?(Regexp)
+
+      message.include?(pattern.to_s)
     end
   end
 end

data/lib/ferrum/downloads.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Tracks files downloaded by a page. Configures the browser's download
+  # behavior/destination via {#set_behavior}, subscribes to the
+  # `Browser.downloadWillBegin`/`Browser.downloadProgress` CDP events to
+  # record their progress in {#files}, and lets callers block until the
+  # current download finishes via {#wait}.
+  #
   class Downloads
     VALID_BEHAVIOR = %i[deny allow allowAndName default].freeze
 
@@ -10,10 +17,26 @@ module Ferrum
       @files = {}
     end
 
+    #
+    # Returns information about all downloaded files.
+    #
+    # @return [Array<Hash>]
+    #
     def files
       @files.values
     end
 
+    #
+    # Waits until the current download finishes.
+    #
+    # @param [Integer] timeout
+    #   How long to wait in seconds.
+    #
+    # @yield
+    #   Optional block that triggers the download, e.g. clicking a link.
+    #
+    # @return [void]
+    #
     def wait(timeout = 5)
       @event.reset
       yield if block_given?
@@ -21,22 +44,47 @@ module Ferrum
       @event.set
     end
 
+    #
+    # Sets the browser's download behavior and destination directory.
+    #
+    # @param [String] save_path
+    #   Absolute path to the directory downloads should be saved to.
+    #
+    # @param [:deny, :allow, :allowAndName, :default] behavior
+    #   Whether/how to allow downloads.
+    #
+    # @return [void]
+    #
+    # @raise [ArgumentError]
+    # @raise [Ferrum::Error]
+    #
     def set_behavior(save_path:, behavior: :allow)
       raise ArgumentError unless VALID_BEHAVIOR.include?(behavior.to_sym)
       raise Error, "supply absolute path for `:save_path` option" unless Pathname.new(save_path.to_s).absolute?
 
+      options = { browserContextId: @page.context_id } if @page.context_id
       @page.command("Browser.setDownloadBehavior",
-                    browserContextId: @page.context_id,
+                    **Hash(options),
                     downloadPath: save_path,
                     behavior: behavior,
                     eventsEnabled: true)
     end
 
+    #
+    # Subscribes to download related CDP events.
+    #
+    # @return [void]
+    #
     def subscribe
       subscribe_download_will_begin
       subscribe_download_progress
     end
 
+    #
+    # Subscribes to the `Browser.downloadWillBegin` event to track new downloads.
+    #
+    # @return [void]
+    #
     def subscribe_download_will_begin
       @page.on("Browser.downloadWillBegin") do |params|
         @event.reset
@@ -44,6 +92,11 @@ module Ferrum
       end
     end
 
+    #
+    # Subscribes to the `Browser.downloadProgress` event to track download state.
+    #
+    # @return [void]
+    #
     def subscribe_download_progress
       @page.on("Browser.downloadProgress") do |params|
         @files[params["guid"]].merge!(params)