ferrum 0.17.2 → 0.19.0

data/lib/ferrum/page/screenshot.rb

@@ -4,12 +4,17 @@ require "ferrum/rgba"
 
 module Ferrum
   class Page
+    #
+    # Captures screenshots, PDFs, and MHTML snapshots of the page, and
+    # exposes viewport/document size helpers used to compute capture areas.
+    #
     module Screenshot
       FULL_WARNING = "Ignoring :selector or :area in #screenshot since full: true was given at %s"
       AREA_WARNING = "Ignoring :area in #screenshot since selector: was given at %s"
 
       DEFAULT_SCREENSHOT_FORMAT = "png"
       SUPPORTED_SCREENSHOT_FORMAT = %w[png jpeg jpg webp].freeze
+      DEFAULT_RENDER_TIMEOUT = 60
 
       DEFAULT_PDF_OPTIONS = {
         landscape: false,
@@ -27,9 +32,9 @@ module Ferrum
         A1: { width: 23.40, height: 33.10 },
         A2: { width: 16.54, height: 23.40 },
         A3: { width: 11.70, height: 16.54 },
-        A4: { width:  8.27, height: 11.70 },
-        A5: { width:  5.83, height:  8.27 },
-        A6: { width:  4.13, height:  5.83 }
+        A4: { width: 8.27, height: 11.70 },
+        A5: { width: 5.83, height: 8.27 },
+        A6: { width: 4.13, height: 5.83 }
       }.freeze
 
       #
@@ -65,6 +70,11 @@ module Ferrum
       # @option opts [Ferrum::RGBA] :background_color
       #   Sets the background color.
       #
+      # @param [Numeric] timeout
+      #   How long to wait for the screenshot to be captured. Defaults to
+      #   {DEFAULT_RENDER_TIMEOUT} since a full-page capture is a known slow
+      #   outlier among CDP commands.
+      #
       # @example
       #   page.go_to("https://google.com/")
       #
@@ -83,10 +93,10 @@ module Ferrum
       # @example Save with specific background color:
       #   page.screenshot(background_color: Ferrum::RGBA.new(0, 0, 0, 0.0))
       #
-      def screenshot(**opts)
+      def screenshot(timeout: DEFAULT_RENDER_TIMEOUT, **opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
-        data = capture_screenshot(options, opts[:full], opts[:background_color])
+        data = capture_screenshot(options, opts[:full], opts[:background_color], timeout)
         return data if encoding == :base64
 
         bin = Base64.decode64(data)
@@ -124,15 +134,20 @@ module Ferrum
       #   See other [native options](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-printToPDF) you
       #   can pass.
       #
+      # @param [Numeric] timeout
+      #   How long to wait for the PDF to be generated. Defaults to
+      #   {DEFAULT_RENDER_TIMEOUT} since large documents are a known slow
+      #   outlier among CDP commands.
+      #
       # @example
       #   page.go_to("https://google.com/")
       #   # Save to disk as a PDF
       #   page.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => true
       #
-      def pdf(**opts)
+      def pdf(timeout: DEFAULT_RENDER_TIMEOUT, **opts)
         path, encoding = common_options(**opts)
         options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
-        handle = command("Page.printToPDF", **options).fetch("stream")
+        handle = command("Page.printToPDF", timeout: timeout, **options).fetch("stream")
         stream_to(path: path, encoding: encoding, handle: handle)
       end
 
@@ -153,18 +168,46 @@ module Ferrum
         save_file(path, data)
       end
 
+      #
+      # Current viewport size.
+      #
+      # @return [(Integer, Integer)]
+      #   The width, height of the viewport.
+      #
+      # @example
+      #   page.viewport_size # => [1024, 768]
+      #
       def viewport_size
         evaluate <<~JS
           [window.innerWidth, window.innerHeight]
         JS
       end
 
+      #
+      # The ratio of the resolution in physical pixels to the resolution in
+      # CSS pixels for the current display device.
+      #
+      # @return [Float]
+      #
+      # @example
+      #   page.device_pixel_ratio # => 1.0
+      #
       def device_pixel_ratio
         evaluate <<~JS
           window.devicePixelRatio
         JS
       end
 
+      #
+      # Full size of the document, including the part that is not visible in
+      # the viewport.
+      #
+      # @return [(Integer, Integer)]
+      #   The scroll width, scroll height of the document.
+      #
+      # @example
+      #   page.document_size # => [1024, 4000]
+      #
       def document_size
         evaluate <<~JS
           [document.documentElement.scrollWidth,
@@ -281,23 +324,12 @@ module Ferrum
         option.to_s.gsub(%r{(?:_|(/))([a-z\d]*)}) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }.to_sym
       end
 
-      def capture_screenshot(options, full, background_color)
-        maybe_resize_fullscreen(full) do
-          with_background_color(background_color) do
-            command("Page.captureScreenshot", **options)
-          end
-        end.fetch("data")
-      end
-
-      def maybe_resize_fullscreen(full)
-        if full
-          width, height = viewport_size.dup
-          resize(fullscreen: true)
-        end
+      def capture_screenshot(options, full, background_color, timeout)
+        options = options.merge(captureBeyondViewport: true) if full
 
-        yield
-      ensure
-        resize(width: width, height: height) if full
+        with_background_color(background_color) do
+          command("Page.captureScreenshot", timeout: timeout, **options)
+        end.fetch("data")
       end
 
       def with_background_color(color)

data/lib/ferrum/page/stream.rb

@@ -2,9 +2,31 @@
 
 module Ferrum
   class Page
+    #
+    # Reads a CDP `IO` stream handle (e.g. from `Page.printToPDF` or
+    # `Tracing.tracingComplete`) in chunks and writes its contents to a file
+    # on disk or accumulates it in memory.
+    #
     module Stream
       STREAM_CHUNK = 128 * 1024
 
+      #
+      # Reads a CDP `IO` stream to a file on disk, or into memory when no
+      # path is given.
+      #
+      # @param [String, nil] path
+      #   The path to save the stream's contents to. When `nil` the contents
+      #   are returned in memory instead.
+      #
+      # @param [Symbol] encoding
+      #   `:base64` or `:binary`. Only used when `path` is `nil`.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [Boolean, String]
+      #   `true` when saved to disk, otherwise the stream's contents.
+      #
       def stream_to(path:, encoding:, handle:)
         if path.nil?
           stream_to_memory(encoding: encoding, handle: handle)
@@ -13,17 +35,52 @@ module Ferrum
         end
       end
 
+      #
+      # Reads a CDP `IO` stream and writes its contents to a file on disk.
+      #
+      # @param [String] path
+      #   The path to save the stream's contents to.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [Boolean]
+      #
       def stream_to_file(path:, handle:)
         File.open(path, "wb") { |f| stream(output: f, handle: handle) }
         true
       end
 
+      #
+      # Reads a CDP `IO` stream into memory.
+      #
+      # @param [Symbol] encoding
+      #   `:base64` to Base64-encode the result, `:binary` to return it as is.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [String]
+      #
       def stream_to_memory(encoding:, handle:)
         data = String.new # Mutable string has << and compatible to File
         stream(output: data, handle: handle)
         encoding == :base64 ? Base64.encode64(data) : data
       end
 
+      #
+      # Reads a CDP `IO` stream in chunks, writing each chunk to the given
+      # output until the stream is exhausted, then releases the stream
+      # handle on the browser side.
+      #
+      # @param [#<<] output
+      #   Anything that responds to `#<<`, e.g. an open `File` or a `String`.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to read from.
+      #
+      # @return [void]
+      #
       def stream(output:, handle:)
         loop do
           result = command("IO.read", handle: handle, size: STREAM_CHUNK)
@@ -32,6 +89,20 @@ module Ferrum
           output << chunk
           break if result["eof"]
         end
+        close_stream(handle: handle)
+      end
+
+      #
+      # Releases a CDP `IO` stream handle, discarding any backing storage
+      # the browser is holding for it.
+      #
+      # @param [String] handle
+      #   The CDP `IO` stream handle to close.
+      #
+      # @return [void]
+      #
+      def close_stream(handle:)
+        command("IO.close", handle: handle)
       end
     end
   end

data/lib/ferrum/page/tracing.rb

@@ -2,6 +2,12 @@
 
 module Ferrum
   class Page
+    #
+    # Records a Chrome performance trace for the page via the CDP
+    # [Tracing](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/)
+    # domain, optionally including screenshots, and streams the resulting
+    # trace data to disk or memory once recording stops.
+    #
     class Tracing
       EXCLUDED_CATEGORIES = %w[*].freeze
       SCREENSHOT_CATEGORIES = %w[disabled-by-default-devtools.screenshot].freeze

data/lib/ferrum/page.rb

@@ -8,6 +8,8 @@ require "ferrum/headers"
 require "ferrum/cookies"
 require "ferrum/dialog"
 require "ferrum/network"
+require "ferrum/accessibility"
+require "ferrum/interceptable"
 require "ferrum/downloads"
 require "ferrum/page/frames"
 require "ferrum/page/screencast"
@@ -17,12 +19,19 @@ require "ferrum/page/tracing"
 require "ferrum/page/stream"
 
 module Ferrum
+  #
+  # Represents a single browser tab (a CDP target of type `page`). Owns the
+  # tab's {Mouse}, {Keyboard}, {Network}, {Cookies}, {Headers}, {Downloads}
+  # and {Accessibility} helpers, as well as its frame tree (see the included
+  # {Page::Frames} module), and is the object that navigation, DOM search and
+  # JavaScript evaluation methods are ultimately delegated to from {Browser}.
+  #
   class Page
     GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
 
     extend Forwardable
 
-    delegate %i[at_css at_xpath css xpath
+    delegate %i[at_css at_xpath css xpath wait_for_selector
                 current_url current_title url title body doctype content=
                 execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
@@ -33,6 +42,7 @@ module Ferrum
     include Screenshot
     include Frames
     include Stream
+    include Interceptable
 
     attr_accessor :referrer
     attr_reader :context_id, :target_id, :event, :tracing
@@ -57,6 +67,11 @@ module Ferrum
     # @return [Network]
     attr_reader :network
 
+    # Accessibility object.
+    #
+    # @return [Accessibility]
+    attr_reader :accessibility
+
     # Headers object.
     #
     # @return [Headers]
@@ -88,6 +103,7 @@ module Ferrum
       @headers = Headers.new(self)
       @cookies = Cookies.new(self)
       @network = Network.new(self)
+      @accessibility = Accessibility.new(self)
       @tracing = Tracing.new(self)
       @downloads = Downloads.new(self)
 
@@ -118,12 +134,20 @@ module Ferrum
     rescue TimeoutError
       if @options.pending_connection_errors
         pendings = network.traffic.select(&:pending?).map(&:url).compact
-        raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
+        raise PendingConnectionsError.new(options[:url], Array(pendings))
       end
     end
     alias goto go_to
     alias go go_to
 
+    #
+    # Closes the page's target and its underlying client connection.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   page.close # => true
+    #
     def close
       @headers.clear
       client.command("Target.closeTarget", async: true, targetId: @target_id)
@@ -132,6 +156,11 @@ module Ferrum
       true
     end
 
+    #
+    # Closes the underlying client connection only, without closing the
+    # target itself. Useful when you want to detach from a page without
+    # ending the browser tab it represents.
+    #
     def close_connection
       client&.close
     end
@@ -147,9 +176,15 @@ module Ferrum
     #
     # @param [Float] scale_factor device scale factor value. 0 disables the override
     #
-    # @param [Boolean] mobile whether to emulate mobile device
+    # @param [Boolean] mobile whether to emulate mobile device and enable touch
     #
-    def set_viewport(width:, height:, scale_factor: 0, mobile: false)
+    def set_viewport(width: 0, height: 0, scale_factor: 0, mobile: false)
+      command(
+        "Emulation.setTouchEmulationEnabled",
+        enabled: mobile,
+        maxTouchPoints: 1
+      )
+
       command(
         "Emulation.setDeviceMetricsOverride",
         slowmoable: true,
@@ -160,6 +195,26 @@ module Ferrum
       )
     end
 
+    #
+    # Resizes the window and emulates the viewport accordingly, optionally
+    # switching to fullscreen.
+    #
+    # @param [Integer, nil] width width value in pixels.
+    #
+    # @param [Integer, nil] height height value in pixels.
+    #
+    # @param [Boolean] fullscreen whether to put the window into fullscreen
+    #   mode. When `true`, `width` and `height` are read from
+    #   {#document_size} instead of the given arguments.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.resize(width: 1024, height: 768)
+    #
+    # @example
+    #   page.resize(fullscreen: true)
+    #
     def resize(width: nil, height: nil, fullscreen: false)
       if fullscreen
         width, height = document_size
@@ -314,6 +369,15 @@ module Ferrum
       history_navigate(delta: 1)
     end
 
+    #
+    # Blocks until the page reloads or the timeout is reached.
+    #
+    # @param [Numeric] timeout
+    #   Maximum time in seconds to wait for a reload event.
+    #
+    # @example
+    #   page.wait_for_reload
+    #
     def wait_for_reload(timeout = 1)
       @event.reset if @event.set?
       @event.wait(timeout)
@@ -353,74 +417,102 @@ module Ferrum
       true
     end
 
-    def command(method, wait: 0, slowmoable: false, **params)
+    #
+    # Sends a CDP command to the browser and optionally waits for network
+    # activity on the main frame to settle before returning.
+    #
+    # @param [String] method
+    #   The CDP method name, e.g. `"Page.navigate"`.
+    #
+    # @param [Numeric] wait
+    #   How many seconds to wait for a network event on the main frame after
+    #   the command is sent. `0` disables waiting.
+    #
+    # @param [Boolean] slowmoable
+    #   Whether to sleep for `Browser::Options#slowmo` seconds before sending
+    #   the command.
+    #
+    # @param [Numeric, nil] timeout
+    #   Overrides the timeout this command's response is bound by. Defaults
+    #   to the page's `timeout`. Callers with their own budget (e.g. `#pdf`/
+    #   `#screenshot`) pass it explicitly.
+    #
+    # @return [Hash{String => Object}]
+    #
+    # @example
+    #   page.command("Page.navigate", url: "https://github.com/")
+    #
+    def command(method, wait: 0, slowmoable: false, timeout: nil, **params)
       iteration = @event.reset if wait.positive?
       sleep(@options.slowmo) if slowmoable && @options.slowmo.positive?
-      result = client.command(method, **params)
+      result = client.command(method, timeout: timeout || self.timeout, **params)
 
       if wait.positive?
         # Wait a bit after command and check if iteration has
-        # changed which means there was some network event for
-        # the main frame and it started to load new content.
+        #  changed, which means there was some network event for
+        # the main frame, and it started to load new content.
         @event.wait(wait)
         if iteration != @event.iteration
-          set = @event.wait(timeout)
+          set = @event.wait(self.timeout)
           raise TimeoutError unless set
         end
       end
       result
     end
 
+    # Subscribes to a CDP event, or to `:dialog`, `:request`, `:response`,
+    # `:auth` (the latter three handled by {Interceptable}).
+    #
+    # @param [Symbol, String] name
+    #
+    # @return [Integer]
+    #   The subscription id, used to unsubscribe via {#off}.
     def on(name, &block)
-      case name
-      when :dialog
-        client.on("Page.javascriptDialogOpening") do |params, index, total|
-          dialog = Dialog.new(self, params)
-          block.call(dialog, index, total)
-        end
-      when :request
-        client.on("Fetch.requestPaused") do |params, index, total|
-          request = Network::InterceptedRequest.new(client, params)
-          exchange = network.select(request.network_id).last
-          exchange ||= network.build_exchange(request.network_id)
-          exchange.intercepted_request = request
-          block.call(request, index, total)
-        end
-      when :auth
-        client.on("Fetch.authRequired") do |params, index, total|
-          request = Network::AuthRequest.new(self, params)
-          block.call(request, index, total)
-        end
-      else
-        client.on(name, &block)
+      return super unless name == :dialog
+
+      client.on("Page.javascriptDialogOpening") do |params, index, total|
+        dialog = Dialog.new(self, params)
+        block.call(dialog, index, total)
       end
     end
 
+    # Unsubscribes a listener previously registered via {#on}.
+    #
+    # @param [Symbol, String] name
+    #
+    # @param [Integer] id
+    #   The subscription id returned by {#on}.
+    #
+    # @return [void]
     def off(name, id)
-      case name
-      when :dialog
-        client.off("Page.javascriptDialogOpening", id)
-      when :request
-        client.off("Fetch.requestPaused", id)
-      when :auth
-        client.off("Fetch.authRequired", id)
-      else
-        client.off(name, id)
-      end
-    end
+      return super unless name == :dialog
 
-    def subscribed?(event)
-      client.subscribed?(event)
+      client.off("Page.javascriptDialogOpening", id)
     end
 
+    # Whether the page is configured to use a proxy.
+    #
+    # @return [Boolean]
     def use_proxy?
       @proxy_host && @proxy_port
     end
 
+    # Whether the page is configured to use a proxy that requires authentication.
+    #
+    # @return [Boolean]
     def use_authorized_proxy?
       use_proxy? && @proxy_user && @proxy_password
     end
 
+    #
+    # Returns the node id of the document's root element.
+    #
+    # @param [Boolean] async
+    #   Whether to send the command without waiting for a response.
+    #
+    # @return [Integer, Boolean]
+    #   The root node id, or `true` when sent asynchronously.
+    #
     def document_node_id(async: false)
       return client.command("DOM.getDocument", async: true, depth: 0) if async
 
@@ -436,7 +528,7 @@ module Ferrum
 
       if @options.logger
         on("Runtime.consoleAPICalled") do |params|
-          params["args"].each { |r| @options.logger.puts(r["value"]) }
+          log_console_api(params)
         end
       end
 
@@ -458,8 +550,9 @@ module Ferrum
 
     def prepare_page
       command("Page.enable")
+      command("Page.setLifecycleEventsEnabled", enabled: true)
       command("Runtime.enable")
-      command("DOM.enable")
+      command("DOM.enable", includeWhitespace: "all")
       command("CSS.enable")
       command("Log.enable")
       command("Network.enable")
@@ -492,6 +585,16 @@ module Ferrum
       document_node_id
     end
 
+    def log_console_api(params)
+      message = params.fetch("args", []).filter_map { |arg| arg["value"] || arg["description"] }.join(" ")
+      @options.logger.puts("[#{params['type']}] #{message}")
+
+      params.dig("stackTrace", "callFrames")&.each do |frame|
+        location = "#{frame['url']}:#{frame['lineNumber'].to_i + 1}:#{frame['columnNumber'].to_i + 1}"
+        @options.logger.puts("    at #{frame['functionName']} (#{location})")
+      end
+    end
+
     def inject_extensions
       @options.extensions.each do |extension|
         # https://github.com/GoogleChrome/puppeteer/issues/1443

data/lib/ferrum/proxy.rb

@@ -1,11 +1,34 @@
 # frozen_string_literal: true
 
 require "tempfile"
-require "webrick"
-require "webrick/httpproxy"
+
+begin
+  require "webrick"
+  require "webrick/httpproxy"
+rescue LoadError
+  warn("Please add webrick to your Gemfile to use Ferrum proxy")
+  raise
+end
 
 module Ferrum
+  #
+  # A local WEBrick-based proxy server, useful for injecting HTTP Basic-Auth
+  # credentials into requests or forwarding to (and rotating between)
+  # upstream proxies, cases the browser's own proxy support can't handle
+  # directly.
+  #
+  # @note Requires the `webrick` gem, which isn't a hard dependency of
+  #   Ferrum; add it to your Gemfile to use this class.
+  #
   class Proxy
+    #
+    # Builds a new proxy server and starts it.
+    #
+    # @param [Hash] args
+    #   Keyword arguments forwarded to {#initialize}.
+    #
+    # @return [Proxy]
+    #
     def self.start(**args)
       new(**args).tap(&:start)
     end
@@ -20,6 +43,11 @@ module Ferrum
       @password = password
     end
 
+    #
+    # Starts the WEBrick proxy server.
+    #
+    # @return [void]
+    #
     def start
       options = {
         ProxyURI: nil, ServerType: Thread,
@@ -45,12 +73,34 @@ module Ferrum
       @port = @server.config[:Port]
     end
 
+    #
+    # Changes the upstream proxy the server forwards connections to.
+    #
+    # @param [String] host
+    #   Address of the upstream proxy.
+    #
+    # @param [Integer] port
+    #   Port of the upstream proxy.
+    #
+    # @param [String, nil] user
+    #   Username for upstream proxy authentication.
+    #
+    # @param [String, nil] password
+    #   Password for upstream proxy authentication.
+    #
+    # @return [void]
+    #
     def rotate(host:, port:, user: nil, password: nil)
       credentials = "#{user}:#{password}@" if user && password
       proxy_uri = "schema://#{credentials}#{host}:#{port}"
       @server.config[:ProxyURI] = URI.parse(proxy_uri)
     end
 
+    #
+    # Stops the proxy server and removes the htpasswd file.
+    #
+    # @return [void]
+    #
     def stop
       @file&.close(true)
       @server.shutdown

data/lib/ferrum/rgba.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
 module Ferrum
+  #
+  # Represents an RGBA color, validating that the red/green/blue components
+  # are integers between 0 and 255 and that alpha is a float between 0.0 and
+  # 1.0. Used e.g. as the background color option for screenshots.
+  #
   class RGBA
     def initialize(red, green, blue, alpha)
       self.red = red
@@ -11,6 +16,11 @@ module Ferrum
       validate
     end
 
+    #
+    # Converts the color to a Hash.
+    #
+    # @return [Hash{Symbol => Integer, Float}]
+    #
     def to_h
       { r: red, g: green, b: blue, a: alpha }
     end