ferrum 0.17.2 → 0.18.0

data/lib/ferrum/page/frames.rb

@@ -4,6 +4,12 @@ require "ferrum/frame"
 
 module Ferrum
   class Page
+    #
+    # Tracks the page's frame tree and keeps it in sync with the browser by
+    # subscribing to the relevant `Page.*`, `Network.*`, and `Runtime.*` CDP
+    # events. Exposes lookups over the tracked {Frame} objects and reports
+    # when the tree has settled into an idle state.
+    #
     module Frames
       # The page's main frame, the top of the tree and the parent of all frames.
       #
@@ -63,12 +69,16 @@ module Ferrum
         end
       end
 
+      private
+
       def frames_subscribe
         subscribe_frame_attached
         subscribe_frame_detached
+        subscribe_frame_started_navigating
         subscribe_frame_started_loading
         subscribe_frame_navigated
         subscribe_frame_stopped_loading
+        subscribe_frame_lifecycle_events
 
         subscribe_navigated_within_document
 
@@ -79,8 +89,6 @@ module Ferrum
         subscribe_execution_contexts_cleared
       end
 
-      private
-
       def subscribe_frame_attached
         on("Page.frameAttached") do |params|
           parent_frame_id, frame_id = params.values_at("parentFrameId", "frameId")
@@ -100,6 +108,18 @@ module Ferrum
         end
       end
 
+      # Tracks the +loaderId+ for each navigating frame and clears stale
+      # lifecycle events from the previous navigation.
+      def subscribe_frame_started_navigating
+        on("Page.frameStartedNavigating") do |params|
+          frame = @frames[params["frameId"]]
+          if frame
+            frame.loader_id = params["loaderId"]
+            frame.lifecycle_events.clear
+          end
+        end
+      end
+
       def subscribe_frame_started_loading
         on("Page.frameStartedLoading") do |params|
           frame = @frames[params["frameId"]]
@@ -138,6 +158,33 @@ module Ferrum
         end
       end
 
+      # Appends +Page.lifecycleEvent+ events to {Frame#lifecycle_events}.
+      # Events from a superseded navigation (+loaderId+ mismatch) are dropped.
+      # Transitions +loading="lazy"+ iframes that Chrome never starts loading
+      # to +:stopped_loading+ on +networkIdle+, preventing a {Page#go_to} timeout.
+      def subscribe_frame_lifecycle_events
+        on("Page.lifecycleEvent") do |params|
+          frame = @frames[params["frameId"]]
+          next unless frame
+
+          frame.loader_id = params["loaderId"] unless frame.loader_id
+          # Reject stale events from a superseded navigation, iframes are not destroyed by Chrome, instead it creates
+          # new ones. Main frame stays with the same id, but new events start to flow in.
+          next if frame.loader_id != params["loaderId"]
+
+          event = params.slice("name", "timestamp")
+          frame.lifecycle_events << event
+
+          # This handles iframes with loading="lazy", those that Chrome attaches but parks outside the viewport.
+          # They do not trigger any events except `Page.frameAttached` and lifecycle events like:
+          # `init` and `networkIdle`. Without it `go_to` would wait for such iframe until it raises timeout.
+          if event["name"] == "networkIdle" && !frame.main?
+            frame.state = :stopped_loading
+            @event.set if idling?
+          end
+        end
+      end
+
       def subscribe_navigated_within_document
         on("Page.navigatedWithinDocument") do
           @event.set if idling?
@@ -177,21 +224,36 @@ module Ferrum
           execution_id = params["executionContextId"]
           frame = frame_by(execution_id: execution_id)
           frame&.execution_id = nil
-          frame&.state = :stopped_loading
+          frame&.state = :canceled
         end
       end
 
+      # On full navigations/reloads Chrome fires +Page.frameAttached+ with new
+      # frame IDs but skips +Page.frameDetached+ for old ones. Removing stale
+      # child frames here prevents them from blocking {#idling?} indefinitely.
+      # The main frame is reset in-place; child frames are re-added via
+      # +Page.frameAttached+.
       def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
-          @frames.each_value do |f|
-            f.execution_id = nil
-            f.state = :stopped_loading
+          children = []
+
+          @frames.each do |frame_id, f|
+            if f.main?
+              f.execution_id = nil
+              f.loader_id = nil
+              f.lifecycle_events.clear
+              f.state = :canceled
+            else
+              children << frame_id
+            end
           end
+
+          children.each { |id| @frames.delete(id) }
         end
       end
 
       def idling?
-        @frames.values.all? { |f| f.state == :stopped_loading }
+        @frames.values.all?(&:idle?)
       end
     end
   end

data/lib/ferrum/page/screencast.rb

@@ -2,6 +2,11 @@
 
 module Ferrum
   class Page
+    #
+    # Starts and stops a live screencast of the page, streaming a sequence
+    # of frame images (a video-like feed) to a given block as the page
+    # renders.
+    #
     module Screencast
       # Starts sending frames to record screencast to the given block.
       #

data/lib/ferrum/page/screenshot.rb

@@ -4,12 +4,17 @@ require "ferrum/rgba"
 
 module Ferrum
   class Page
+    #
+    # Captures screenshots, PDFs, and MHTML snapshots of the page, and
+    # exposes viewport/document size helpers used to compute capture areas.
+    #
     module Screenshot
       FULL_WARNING = "Ignoring :selector or :area in #screenshot since full: true was given at %s"
       AREA_WARNING = "Ignoring :area in #screenshot since selector: was given at %s"
 
       DEFAULT_SCREENSHOT_FORMAT = "png"
       SUPPORTED_SCREENSHOT_FORMAT = %w[png jpeg jpg webp].freeze
+      DEFAULT_RENDER_TIMEOUT = 60
 
       DEFAULT_PDF_OPTIONS = {
         landscape: false,
@@ -65,6 +70,11 @@ module Ferrum
       # @option opts [Ferrum::RGBA] :background_color
       #   Sets the background color.
       #
+      # @param [Numeric] timeout
+      #   How long to wait for the screenshot to be captured. Defaults to
+      #   {DEFAULT_RENDER_TIMEOUT} since a full-page capture is a known slow
+      #   outlier among CDP commands.
+      #
       # @example
       #   page.go_to("https://google.com/")
       #
@@ -83,10 +93,10 @@ module Ferrum
       # @example Save with specific background color:
       #   page.screenshot(background_color: Ferrum::RGBA.new(0, 0, 0, 0.0))
       #
-      def screenshot(**opts)
+      def screenshot(timeout: DEFAULT_RENDER_TIMEOUT, **opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
-        data = capture_screenshot(options, opts[:full], opts[:background_color])
+        data = capture_screenshot(options, opts[:full], opts[:background_color], timeout)
         return data if encoding == :base64
 
         bin = Base64.decode64(data)
@@ -124,15 +134,20 @@ module Ferrum
       #   See other [native options](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-printToPDF) you
       #   can pass.
       #
+      # @param [Numeric] timeout
+      #   How long to wait for the PDF to be generated. Defaults to
+      #   {DEFAULT_RENDER_TIMEOUT} since large documents are a known slow
+      #   outlier among CDP commands.
+      #
       # @example
       #   page.go_to("https://google.com/")
       #   # Save to disk as a PDF
       #   page.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => true
       #
-      def pdf(**opts)
+      def pdf(timeout: DEFAULT_RENDER_TIMEOUT, **opts)
         path, encoding = common_options(**opts)
         options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
-        handle = command("Page.printToPDF", **options).fetch("stream")
+        handle = command("Page.printToPDF", timeout: timeout, **options).fetch("stream")
         stream_to(path: path, encoding: encoding, handle: handle)
       end
 
@@ -153,18 +168,46 @@ module Ferrum
         save_file(path, data)
       end
 
+      #
+      # Current viewport size.
+      #
+      # @return [(Integer, Integer)]
+      #   The width, height of the viewport.
+      #
+      # @example
+      #   page.viewport_size # => [1024, 768]
+      #
       def viewport_size
         evaluate <<~JS
           [window.innerWidth, window.innerHeight]
         JS
       end
 
+      #
+      # The ratio of the resolution in physical pixels to the resolution in
+      # CSS pixels for the current display device.
+      #
+      # @return [Float]
+      #
+      # @example
+      #   page.device_pixel_ratio # => 1.0
+      #
       def device_pixel_ratio
         evaluate <<~JS
           window.devicePixelRatio
         JS
       end
 
+      #
+      # Full size of the document, including the part that is not visible in
+      # the viewport.
+      #
+      # @return [(Integer, Integer)]
+      #   The scroll width, scroll height of the document.
+      #
+      # @example
+      #   page.document_size # => [1024, 4000]
+      #
       def document_size
         evaluate <<~JS
           [document.documentElement.scrollWidth,
@@ -281,23 +324,12 @@ module Ferrum
         option.to_s.gsub(%r{(?:_|(/))([a-z\d]*)}) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }.to_sym
       end
 
-      def capture_screenshot(options, full, background_color)
-        maybe_resize_fullscreen(full) do
-          with_background_color(background_color) do
-            command("Page.captureScreenshot", **options)
-          end
-        end.fetch("data")
-      end
-
-      def maybe_resize_fullscreen(full)
-        if full
-          width, height = viewport_size.dup
-          resize(fullscreen: true)
-        end
+      def capture_screenshot(options, full, background_color, timeout)
+        options = options.merge(captureBeyondViewport: true) if full
 
-        yield
-      ensure
-        resize(width: width, height: height) if full
+        with_background_color(background_color) do
+          command("Page.captureScreenshot", timeout: timeout, **options)
+        end.fetch("data")
       end
 
       def with_background_color(color)

data/lib/ferrum/page/stream.rb

@@ -2,9 +2,31 @@
 
 module Ferrum
   class Page
+    #
+    # Reads a CDP `IO` stream handle (e.g. from `Page.printToPDF` or
+    # `Tracing.tracingComplete`) in chunks and writes its contents to a file
+    # on disk or accumulates it in memory.
+    #
     module Stream
       STREAM_CHUNK = 128 * 1024
 
+      #
+      # Reads a CDP `IO` stream to a file on disk, or into memory when no
+      # path is given.
+      #
+      # @param [String, nil] path
+      #   The path to save the stream's contents to. When `nil` the contents
+      #   are returned in memory instead.
+      #
+      # @param [Symbol] encoding
+      #   `:base64` or `:binary`. Only used when `path` is `nil`.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [Boolean, String]
+      #   `true` when saved to disk, otherwise the stream's contents.
+      #
       def stream_to(path:, encoding:, handle:)
         if path.nil?
           stream_to_memory(encoding: encoding, handle: handle)
@@ -13,17 +35,51 @@ module Ferrum
         end
       end
 
+      #
+      # Reads a CDP `IO` stream and writes its contents to a file on disk.
+      #
+      # @param [String] path
+      #   The path to save the stream's contents to.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [Boolean]
+      #
       def stream_to_file(path:, handle:)
         File.open(path, "wb") { |f| stream(output: f, handle: handle) }
         true
       end
 
+      #
+      # Reads a CDP `IO` stream into memory.
+      #
+      # @param [Symbol] encoding
+      #   `:base64` to Base64-encode the result, `:binary` to return it as is.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [String]
+      #
       def stream_to_memory(encoding:, handle:)
         data = String.new # Mutable string has << and compatible to File
         stream(output: data, handle: handle)
         encoding == :base64 ? Base64.encode64(data) : data
       end
 
+      #
+      # Reads a CDP `IO` stream in chunks, writing each chunk to the given
+      # output until the stream is exhausted.
+      #
+      # @param [#<<] output
+      #   Anything that responds to `#<<`, e.g. an open `File` or a `String`.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [void]
+      #
       def stream(output:, handle:)
         loop do
           result = command("IO.read", handle: handle, size: STREAM_CHUNK)

data/lib/ferrum/page/tracing.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Page
+    #
+    # Records a Chrome performance trace for the page via the CDP
+    # [Tracing](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/)
+    # domain, optionally including screenshots, and streams the resulting
+    # trace data to disk or memory once recording stops.
+    #
     class Tracing
       EXCLUDED_CATEGORIES = %w[*].freeze
       SCREENSHOT_CATEGORIES = %w[disabled-by-default-devtools.screenshot].freeze

data/lib/ferrum/page.rb

@@ -8,6 +8,8 @@ require "ferrum/headers"
 require "ferrum/cookies"
 require "ferrum/dialog"
 require "ferrum/network"
+require "ferrum/accessibility"
+require "ferrum/interceptable"
 require "ferrum/downloads"
 require "ferrum/page/frames"
 require "ferrum/page/screencast"
@@ -17,6 +19,13 @@ require "ferrum/page/tracing"
 require "ferrum/page/stream"
 
 module Ferrum
+  #
+  # Represents a single browser tab (a CDP target of type `page`). Owns the
+  # tab's {Mouse}, {Keyboard}, {Network}, {Cookies}, {Headers}, {Downloads}
+  # and {Accessibility} helpers, as well as its frame tree (see the included
+  # {Page::Frames} module), and is the object that navigation, DOM search and
+  # JavaScript evaluation methods are ultimately delegated to from {Browser}.
+  #
   class Page
     GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
 
@@ -33,6 +42,7 @@ module Ferrum
     include Screenshot
     include Frames
     include Stream
+    include Interceptable
 
     attr_accessor :referrer
     attr_reader :context_id, :target_id, :event, :tracing
@@ -57,6 +67,11 @@ module Ferrum
     # @return [Network]
     attr_reader :network
 
+    # Accessibility object.
+    #
+    # @return [Accessibility]
+    attr_reader :accessibility
+
     # Headers object.
     #
     # @return [Headers]
@@ -88,6 +103,7 @@ module Ferrum
       @headers = Headers.new(self)
       @cookies = Cookies.new(self)
       @network = Network.new(self)
+      @accessibility = Accessibility.new(self)
       @tracing = Tracing.new(self)
       @downloads = Downloads.new(self)
 
@@ -118,12 +134,20 @@ module Ferrum
     rescue TimeoutError
       if @options.pending_connection_errors
         pendings = network.traffic.select(&:pending?).map(&:url).compact
-        raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
+        raise PendingConnectionsError.new(options[:url], Array(pendings))
       end
     end
     alias goto go_to
     alias go go_to
 
+    #
+    # Closes the page's target and its underlying client connection.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   page.close # => true
+    #
     def close
       @headers.clear
       client.command("Target.closeTarget", async: true, targetId: @target_id)
@@ -132,6 +156,11 @@ module Ferrum
       true
     end
 
+    #
+    # Closes the underlying client connection only, without closing the
+    # target itself. Useful when you want to detach from a page without
+    # ending the browser tab it represents.
+    #
     def close_connection
       client&.close
     end
@@ -160,6 +189,26 @@ module Ferrum
       )
     end
 
+    #
+    # Resizes the window and emulates the viewport accordingly, optionally
+    # switching to fullscreen.
+    #
+    # @param [Integer, nil] width width value in pixels.
+    #
+    # @param [Integer, nil] height height value in pixels.
+    #
+    # @param [Boolean] fullscreen whether to put the window into fullscreen
+    #   mode. When `true`, `width` and `height` are read from
+    #   {#document_size} instead of the given arguments.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.resize(width: 1024, height: 768)
+    #
+    # @example
+    #   page.resize(fullscreen: true)
+    #
     def resize(width: nil, height: nil, fullscreen: false)
       if fullscreen
         width, height = document_size
@@ -314,6 +363,15 @@ module Ferrum
       history_navigate(delta: 1)
     end
 
+    #
+    # Blocks until the page reloads or the timeout is reached.
+    #
+    # @param [Numeric] timeout
+    #   Maximum time in seconds to wait for a reload event.
+    #
+    # @example
+    #   page.wait_for_reload
+    #
     def wait_for_reload(timeout = 1)
       @event.reset if @event.set?
       @event.wait(timeout)
@@ -353,74 +411,102 @@ module Ferrum
       true
     end
 
-    def command(method, wait: 0, slowmoable: false, **params)
+    #
+    # Sends a CDP command to the browser and optionally waits for network
+    # activity on the main frame to settle before returning.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Page.navigate"`.
+    #
+    # @param [Numeric] wait
+    #   How many seconds to wait for a network event on the main frame after
+    #   the command is sent. `0` disables waiting.
+    #
+    # @param [Boolean] slowmoable
+    #   Whether to sleep for `Browser::Options#slowmo` seconds before sending
+    #   the command.
+    #
+    # @param [Numeric, nil] timeout
+    #   Overrides the timeout this command's response is bound by. Defaults
+    #   to the page's `timeout`. Callers with their own budget (e.g. `#pdf`/
+    #   `#screenshot`) pass it explicitly.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.command("Page.navigate", url: "https://github.com/")
+    #
+    def command(method, wait: 0, slowmoable: false, timeout: nil, **params)
       iteration = @event.reset if wait.positive?
       sleep(@options.slowmo) if slowmoable && @options.slowmo.positive?
-      result = client.command(method, **params)
+      result = client.command(method, timeout: timeout || self.timeout, **params)
 
       if wait.positive?
         # Wait a bit after command and check if iteration has
-        # changed which means there was some network event for
-        # the main frame and it started to load new content.
+        #  changed, which means there was some network event for
+        # the main frame, and it started to load new content.
         @event.wait(wait)
         if iteration != @event.iteration
-          set = @event.wait(timeout)
+          set = @event.wait(self.timeout)
           raise TimeoutError unless set
         end
       end
       result
     end
 
+    # Subscribes to a CDP event, or to `:dialog`, `:request`, `:auth` (the
+    # latter two handled by {Interceptable}).
+    #
+    # @param [Symbol, String] name
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
     def on(name, &block)
-      case name
-      when :dialog
-        client.on("Page.javascriptDialogOpening") do |params, index, total|
-          dialog = Dialog.new(self, params)
-          block.call(dialog, index, total)
-        end
-      when :request
-        client.on("Fetch.requestPaused") do |params, index, total|
-          request = Network::InterceptedRequest.new(client, params)
-          exchange = network.select(request.network_id).last
-          exchange ||= network.build_exchange(request.network_id)
-          exchange.intercepted_request = request
-          block.call(request, index, total)
-        end
-      when :auth
-        client.on("Fetch.authRequired") do |params, index, total|
-          request = Network::AuthRequest.new(self, params)
-          block.call(request, index, total)
-        end
-      else
-        client.on(name, &block)
+      return super unless name == :dialog
+
+      client.on("Page.javascriptDialogOpening") do |params, index, total|
+        dialog = Dialog.new(self, params)
+        block.call(dialog, index, total)
       end
     end
 
+    # Unsubscribes a listener previously registered via {#on}.
+    #
+    # @param [Symbol, String] name
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
     def off(name, id)
-      case name
-      when :dialog
-        client.off("Page.javascriptDialogOpening", id)
-      when :request
-        client.off("Fetch.requestPaused", id)
-      when :auth
-        client.off("Fetch.authRequired", id)
-      else
-        client.off(name, id)
-      end
-    end
+      return super unless name == :dialog
 
-    def subscribed?(event)
-      client.subscribed?(event)
+      client.off("Page.javascriptDialogOpening", id)
     end
 
+    # Whether the page is configured to use a proxy.
+    #
+    # @return [Boolean]
     def use_proxy?
       @proxy_host && @proxy_port
     end
 
+    # Whether the page is configured to use a proxy that requires authentication.
+    #
+    # @return [Boolean]
     def use_authorized_proxy?
       use_proxy? && @proxy_user && @proxy_password
     end
 
+    #
+    # Returns the node id of the document's root element.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @return [Integer, Boolean]
+    #   The root node id, or `true` when sent asynchronously.
+    #
     def document_node_id(async: false)
       return client.command("DOM.getDocument", async: true, depth: 0) if async
 
@@ -436,7 +522,7 @@ module Ferrum
 
       if @options.logger
         on("Runtime.consoleAPICalled") do |params|
-          params["args"].each { |r| @options.logger.puts(r["value"]) }
+          log_console_api(params)
         end
       end
 
@@ -458,8 +544,9 @@ module Ferrum
 
     def prepare_page
       command("Page.enable")
+      command("Page.setLifecycleEventsEnabled", enabled: true)
       command("Runtime.enable")
-      command("DOM.enable")
+      command("DOM.enable", includeWhitespace: "all")
       command("CSS.enable")
       command("Log.enable")
       command("Network.enable")
@@ -492,6 +579,16 @@ module Ferrum
       document_node_id
     end
 
+    def log_console_api(params)
+      message = params.fetch("args", []).filter_map { |arg| arg["value"] || arg["description"] }.join(" ")
+      @options.logger.puts("[#{params['type']}] #{message}")
+
+      params.dig("stackTrace", "callFrames")&.each do |frame|
+        location = "#{frame['url']}:#{frame['lineNumber'].to_i + 1}:#{frame['columnNumber'].to_i + 1}"
+        @options.logger.puts("    at #{frame['functionName']} (#{location})")
+      end
+    end
+
     def inject_extensions
       @options.extensions.each do |extension|
         # https://github.com/GoogleChrome/puppeteer/issues/1443