ferrum 0.13 → 0.15

data/lib/ferrum/network/intercepted_request.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
+require "ferrum/network/request_params"
 require "base64"
 
 module Ferrum
   class Network
     class InterceptedRequest
+      include RequestParams
+
       attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
 
-      def initialize(page, params)
+      def initialize(client, params)
         @status = nil
-        @page = page
+        @client = client
         @params = params
         @request_id = params["requestId"]
         @frame_id = params["frameId"]
@@ -40,30 +43,18 @@ module Ferrum
         options = options.merge(body: Base64.strict_encode64(options.fetch(:body, ""))) if has_body
 
         @status = :responded
-        @page.command("Fetch.fulfillRequest", **options)
+        @client.command("Fetch.fulfillRequest", async: true, **options)
       end
 
       def continue(**options)
         options = options.merge(requestId: request_id)
         @status = :continued
-        @page.command("Fetch.continueRequest", **options)
+        @client.command("Fetch.continueRequest", async: true, **options)
       end
 
       def abort
         @status = :aborted
-        @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
-      end
-
-      def url
-        @request["url"]
-      end
-
-      def method
-        @request["method"]
-      end
-
-      def headers
-        @request["headers"]
+        @client.command("Fetch.failRequest", async: true, requestId: request_id, errorReason: "BlockedByClient")
       end
 
       def initial_priority

data/lib/ferrum/network/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "ferrum/network/request_params"
 require "time"
 
 module Ferrum
@@ -9,6 +10,8 @@ module Ferrum
     # object.
     #
     class Request
+      include RequestParams
+
       #
       # Initializes the request object.
       #
@@ -51,69 +54,50 @@ module Ferrum
       end
 
       #
-      # The frame ID of the request.
-      #
-      # @return [String]
-      #
-      def frame_id
-        @params["frameId"]
-      end
-
-      #
-      # The URL for the request.
-      #
-      # @return [String]
-      #
-      def url
-        @request["url"]
-      end
-
+      # Determines if the request is XHR.
       #
-      # The URL fragment for the request.
-      #
-      # @return [String, nil]
+      # @return [Boolean]
       #
-      def url_fragment
-        @request["urlFragment"]
+      def xhr?
+        type?("xhr")
       end
 
       #
-      # The request method.
+      # The frame ID of the request.
       #
       # @return [String]
       #
-      def method
-        @request["method"]
+      def frame_id
+        @params["frameId"]
       end
 
       #
-      # The request headers.
+      # The request timestamp.
       #
-      # @return [Hash{String => String}]
+      # @return [Time]
       #
-      def headers
-        @request["headers"]
+      def time
+        @time ||= Time.strptime(@params["wallTime"].to_s, "%s")
       end
 
       #
-      # The request timestamp.
+      # Determines if a request is of type ping.
       #
-      # @return [Time]
+      # @return [Boolean]
       #
-      def time
-        @time ||= Time.strptime(@params["wallTime"].to_s, "%s")
+      def ping?
+        type?("ping")
       end
 
       #
-      # The optional HTTP `POST` form data.
+      # Converts the request to a Hash.
       #
-      # @return [String, nil]
-      #   The HTTP `POST` form data.
+      # @return [Hash{String => Object}]
+      #   The params of the request.
       #
-      def post_data
-        @request["postData"]
+      def to_h
+        @params
       end
-      alias body post_data
     end
   end
 end

data/lib/ferrum/network/request_params.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Network
+    #
+    # Common methods used by both {Request} and {InterceptedRequest}.
+    #
+    module RequestParams
+      #
+      # The URL for the request.
+      #
+      # @return [String]
+      #
+      def url
+        @request["url"]
+      end
+
+      #
+      # The URL fragment for the request.
+      #
+      # @return [String, nil]
+      #
+      def url_fragment
+        @request["urlFragment"]
+      end
+
+      #
+      # The request method.
+      #
+      # @return [String]
+      #
+      def method
+        @request["method"]
+      end
+
+      #
+      # The request headers.
+      #
+      # @return [Hash{String => String}]
+      #
+      def headers
+        @request["headers"]
+      end
+
+      #
+      # The optional HTTP `POST` form data.
+      #
+      # @return [String, nil]
+      #   The HTTP `POST` form data.
+      #
+      def post_data
+        @request["postData"]
+      end
+      alias body post_data
+    end
+  end
+end

data/lib/ferrum/network/response.rb

@@ -18,8 +18,13 @@ module Ferrum
       # @return [Hash{String => Object}]
       attr_reader :params
 
+      # The response is fully loaded by the browser.
       #
-      # Initializes the respones object.
+      # @return [Boolean]
+      attr_writer :loaded
+
+      #
+      # Initializes the responses object.
       #
       # @param [Page] page
       #   The page associated with the network response.
@@ -121,9 +126,8 @@ module Ferrum
       #
       def body
         @body ||= begin
-          body, encoded = @page
-                          .command("Network.getResponseBody", requestId: id)
-                          .values_at("body", "base64Encoded")
+          body, encoded = @page.command("Network.getResponseBody", requestId: id)
+                               .values_at("body", "base64Encoded")
           encoded ? Base64.decode64(body) : body
         end
       end
@@ -135,8 +139,22 @@ module Ferrum
         @page.network.response == self
       end
 
+      # The response is fully loaded by the browser or not.
+      #
+      # @return [Boolean]
+      def loaded?
+        @loaded
+      end
+
+      # Whether the response is a redirect.
+      #
+      # @return [Boolean]
+      def redirect?
+        params.key?("redirectResponse")
+      end
+
       #
-      # Comapres the respones ID to another response's ID.
+      # Compares the response's ID to another response's ID.
       #
       # @return [Boolean]
       #   Indicates whether the response has the same ID as the other response
@@ -154,6 +172,8 @@ module Ferrum
       def inspect
         %(#<#{self.class} @params=#{@params.inspect} @response=#{@response.inspect}>)
       end
+
+      alias to_h params
     end
   end
 end

data/lib/ferrum/network.rb

@@ -11,9 +11,10 @@ module Ferrum
   class Network
     CLEAR_TYPE = %i[traffic cache].freeze
     AUTHORIZE_TYPE = %i[server proxy].freeze
-    RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
-                        XHR Fetch EventSource WebSocket Manifest
-                        SignedExchange Ping CSPViolationReport Other].freeze
+    REQUEST_STAGES = %i[Request Response].freeze
+    RESOURCE_TYPES = %i[Document Stylesheet Image Media Font Script TextTrack
+                        XHR Fetch Prefetch EventSource WebSocket Manifest
+                        SignedExchange Ping CSPViolationReport Preflight Other].freeze
     AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
                               "or subscribe to `on(:request)` events before calling it"
     AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
@@ -58,7 +59,7 @@ module Ferrum
     #   browser.at_xpath("//a[text() = 'No UI changes button']").click
     #   browser.network.wait_for_idle
     #
-    def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
+    def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.timeout)
       start = Utils::ElapsedTime.monotonic_time
 
       until idle?(connections)
@@ -187,11 +188,20 @@ module Ferrum
     #   end
     #   browser.go_to("https://google.com")
     #
-    def intercept(pattern: "*", resource_type: nil)
+    def intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true)
       pattern = { urlPattern: pattern }
-      pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
 
-      @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
+      if resource_type && RESOURCE_TYPES.none?(resource_type.to_sym)
+        raise ArgumentError, "Unknown resource type '#{resource_type}' must be #{RESOURCE_TYPES.join(' | ')}"
+      end
+
+      if request_stage && REQUEST_STAGES.none?(request_stage.to_sym)
+        raise ArgumentError, "Unknown request stage '#{request_stage}' must be #{REQUEST_STAGES.join(' | ')}"
+      end
+
+      pattern[:resourceType] = resource_type if resource_type
+      pattern[:requestStage] = request_stage if request_stage
+      @page.command("Fetch.enable", patterns: [pattern], handleAuthRequests: handle_auth_requests)
     end
 
     #
@@ -323,13 +333,23 @@ module Ferrum
     #
     # @example
     #   browser.network.offline_mode
-    #   browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed to reach
-    #   server, check DNS and server status)
+    #   browser.go_to("https://github.com/")
+    #     # => Request to https://github.com/ failed (net::ERR_INTERNET_DISCONNECTED) (Ferrum::StatusError)
     #
     def offline_mode
       emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
     end
 
+    #
+    # Toggles ignoring cache for each request. If true, cache will not be used.
+    #
+    # @example
+    #   browser.network.cache(disable: true)
+    #
+    def cache(disable:)
+      @page.command("Network.setCacheDisabled", cacheDisabled: disable)
+    end
+
     private
 
     def subscribe_request_will_be_sent
@@ -352,6 +372,7 @@ module Ferrum
         if params["redirectResponse"]
           previous_exchange = select(request.id)[-2]
           response = Network::Response.new(@page, params)
+          response.loaded = true
           previous_exchange.response = response
         end
 
@@ -364,26 +385,31 @@ module Ferrum
     def subscribe_response_received
       @page.on("Network.responseReceived") do |params|
         exchange = select(params["requestId"]).last
+        next unless exchange
 
-        if exchange
-          response = Network::Response.new(@page, params)
-          exchange.response = response
-        end
+        response = Network::Response.new(@page, params)
+        exchange.response = response
       end
     end
 
     def subscribe_loading_finished
       @page.on("Network.loadingFinished") do |params|
         exchange = select(params["requestId"]).last
-        exchange.response.body_size = params["encodedDataLength"] if exchange&.response
+        next unless exchange
+
+        if (response = exchange.response)
+          response.loaded = true
+          response.body_size = params["encodedDataLength"]
+        end
       end
     end
 
     def subscribe_loading_failed
       @page.on("Network.loadingFailed") do |params|
         exchange = select(params["requestId"]).last
-        exchange.error ||= Network::Error.new
+        next unless exchange
 
+        exchange.error ||= Network::Error.new
         exchange.error.id = params["requestId"]
         exchange.error.type = params["type"]
         exchange.error.error_text = params["errorText"]
@@ -397,8 +423,9 @@ module Ferrum
         entry = params["entry"] || {}
         if entry["source"] == "network" && entry["level"] == "error"
           exchange = select(entry["networkRequestId"]).last
-          exchange.error ||= Network::Error.new
+          next unless exchange
 
+          exchange.error ||= Network::Error.new
           exchange.error.id = entry["networkRequestId"]
           exchange.error.url = entry["url"]
           exchange.error.description = entry["text"]

data/lib/ferrum/node.rb

@@ -88,6 +88,26 @@ module Ferrum
       raise NotImplementedError
     end
 
+    def scroll_into_view
+      tap { page.command("DOM.scrollIntoViewIfNeeded", nodeId: node_id) }
+    end
+
+    def in_viewport?(of: nil)
+      function = <<~JS
+        function(element, scope) {
+          const rect = element.getBoundingClientRect();
+          const [height, width] = scope
+            ? [scope.offsetHeight, scope.offsetWidth]
+            : [window.innerHeight, window.innerWidth];
+          return rect.top >= 0 &&
+           rect.left >= 0 &&
+           rect.bottom <= height &&
+           rect.right <= width;
+        }
+      JS
+      page.evaluate_func(function, self, of)
+    end
+
     def select_file(value)
       page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
     end
@@ -208,7 +228,7 @@ module Ferrum
 
     def content_quads
       quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
-      raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size.zero?
+      raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.empty?
 
       quads
     end

data/lib/ferrum/page/frames.rb

@@ -16,8 +16,8 @@ module Ferrum
       # @return [Array<Frame>]
       #
       # @example
-      #   browser.go_to("https://www.w3schools.com/tags/tag_frame.asp")
-      #   browser.frames # =>
+      #   page.go_to("https://www.w3schools.com/tags/tag_frame.asp")
+      #   page.frames # =>
       #   # [
       #   #   #<Ferrum::Frame
       #   #     @id="C6D104CE454A025FBCF22B98DE612B12"
@@ -39,7 +39,7 @@ module Ferrum
       # Find frame by given options.
       #
       # @param [String] id
-      #   Unique frame's id that browser provides.
+      #   Unique frame's id that page provides.
       #
       # @param [String] name
       #   Frame's name if there's one.
@@ -51,7 +51,7 @@ module Ferrum
       #   The matching frame.
       #
       # @example
-      #   browser.frame_by(id: "C6D104CE454A025FBCF22B98DE612B12")
+      #   page.frame_by(id: "C6D104CE454A025FBCF22B98DE612B12")
       #
       def frame_by(id: nil, name: nil, execution_id: nil)
         if id
@@ -134,7 +134,7 @@ module Ferrum
           end
 
           frame = @frames[params["frameId"]]
-          frame.state = :stopped_loading
+          frame&.state = :stopped_loading
 
           @event.set if idling?
         end

data/lib/ferrum/page/screenshot.rb

@@ -5,6 +5,9 @@ require "ferrum/rgba"
 module Ferrum
   class Page
     module Screenshot
+      FULL_WARNING = "Ignoring :selector or :area in #screenshot since full: true was given at %s"
+      AREA_WARNING = "Ignoring :area in #screenshot since selector: was given at %s"
+
       DEFAULT_PDF_OPTIONS = {
         landscape: false,
         paper_width: 8.5,
@@ -50,6 +53,9 @@ module Ferrum
       # @option opts [String] :selector
       #   CSS selector for the given element.
       #
+      # @option opts [Hash] :area
+      #   x, y, width, height to screenshot an area.
+      #
       # @option opts [Float] :scale
       #   Zoom in/out.
       #
@@ -57,19 +63,19 @@ module Ferrum
       #   Sets the background color.
       #
       # @example
-      #   browser.go_to("https://google.com/")
+      #   page.go_to("https://google.com/")
       #
       # @example Save on the disk in PNG:
-      #   browser.screenshot(path: "google.png") # => 134660
+      #   page.screenshot(path: "google.png") # => 134660
       #
       # @example Save on the disk in JPG:
-      #   browser.screenshot(path: "google.jpg") # => 30902
+      #   page.screenshot(path: "google.jpg") # => 30902
       #
       # @example Save to Base64 the whole page not only viewport and reduce quality:
-      #   browser.screenshot(full: true, quality: 60) # "iVBORw0KGgoAAAANS...
+      #   page.screenshot(full: true, quality: 60) # "iVBORw0KGgoAAAANS...
       #
       # @example Save with specific background color:
-      #   browser.screenshot(background_color: Ferrum::RGBA.new(0, 0, 0, 0.0))
+      #   page.screenshot(background_color: Ferrum::RGBA.new(0, 0, 0, 0.0))
       #
       def screenshot(**opts)
         path, encoding = common_options(**opts)
@@ -113,9 +119,9 @@ module Ferrum
       #   can pass.
       #
       # @example
-      #   browser.go_to("https://google.com/")
+      #   page.go_to("https://google.com/")
       #   # Save to disk as a PDF
-      #   browser.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => true
+      #   page.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => true
       #
       def pdf(**opts)
         path, encoding = common_options(**opts)
@@ -131,8 +137,8 @@ module Ferrum
       #   The path to save a file on the disk.
       #
       # @example
-      #   browser.go_to("https://google.com/")
-      #   browser.mhtml(path: "google.mhtml") # => 87742
+      #   page.go_to("https://google.com/")
+      #   page.mhtml(path: "google.mhtml") # => 87742
       #
       def mhtml(path: nil)
         data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
@@ -147,6 +153,12 @@ module Ferrum
         JS
       end
 
+      def device_pixel_ratio
+        evaluate <<~JS
+          window.devicePixelRatio
+        JS
+      end
+
       def document_size
         evaluate <<~JS
           [document.documentElement.scrollWidth,
@@ -192,7 +204,7 @@ module Ferrum
         screenshot_options.merge!(quality: quality) if quality
         screenshot_options.merge!(format: format)
 
-        clip = area_options(options[:full], options[:selector], scale)
+        clip = area_options(options[:full], options[:selector], scale, options[:area])
         screenshot_options.merge!(clip: clip) if clip
 
         screenshot_options
@@ -208,29 +220,35 @@ module Ferrum
         [format, quality]
       end
 
-      def area_options(full, selector, scale)
-        message = "Ignoring :selector in #screenshot since full: true was given at #{caller(1..1).first}"
-        warn(message) if full && selector
+      def area_options(full, selector, scale, area = nil)
+        warn(FULL_WARNING % caller(1..1).first) if full && (selector || area)
+        warn(AREA_WARNING % caller(1..1).first) if selector && area
 
         clip = if full
-                 width, height = document_size
-                 { x: 0, y: 0, width: width, height: height, scale: scale } if width.positive? && height.positive?
+                 full_window_area || viewport_area
                elsif selector
-                 bounding_rect(selector).merge(scale: scale)
+                 bounding_rect(selector)
+               elsif area
+                 area
+               else
+                 viewport_area
                end
 
-        if scale != 1
-          unless clip
-            width, height = viewport_size
-            clip = { x: 0, y: 0, width: width, height: height }
-          end
-
-          clip.merge!(scale: scale)
-        end
+        clip.merge!(scale: scale)
 
         clip
       end
 
+      def full_window_area
+        width, height = document_size
+        { x: 0, y: 0, width: width, height: height } if width.positive? && height.positive?
+      end
+
+      def viewport_area
+        width, height = viewport_size
+        { x: 0, y: 0, width: width, height: height }
+      end
+
       def bounding_rect(selector)
         rect = evaluate_async(%(
           const rect = document