ferrum 0.10.1 → 0.12

data/lib/ferrum/network.rb

@@ -14,6 +14,10 @@ module Ferrum
     RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
                         XHR Fetch EventSource WebSocket Manifest
                         SignedExchange Ping CSPViolationReport Other].freeze
+    AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
+                              "or subscribe to `on(:request)` events before calling it"
+    AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
+    ALLOWED_CONNECTION_TYPE = %w[none cellular2g cellular3g cellular4g bluetooth ethernet wifi wimax other].freeze
 
     attr_reader :traffic
 
@@ -21,13 +25,16 @@ module Ferrum
       @page = page
       @traffic = []
       @exchange = nil
+      @blacklist = nil
+      @whitelist = nil
     end
 
     def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
-      start = Ferrum.monotonic_time
+      start = Utils::ElapsedTime.monotonic_time
 
       until idle?(connections)
-        raise TimeoutError if Ferrum.timeout?(start, timeout)
+        raise TimeoutError if Utils::ElapsedTime.timeout?(start, timeout)
+
         sleep(duration)
       end
     end
@@ -61,9 +68,7 @@ module Ferrum
     end
 
     def clear(type)
-      unless CLEAR_TYPE.include?(type)
-        raise ArgumentError, ":type should be in #{CLEAR_TYPE}"
-      end
+      raise ArgumentError, ":type should be in #{CLEAR_TYPE}" unless CLEAR_TYPE.include?(type)
 
       if type == :traffic
         @traffic.clear
@@ -74,23 +79,28 @@ module Ferrum
       true
     end
 
+    def blacklist=(patterns)
+      @blacklist = Array(patterns)
+      blacklist_subscribe
+    end
+    alias blocklist= blacklist=
+
+    def whitelist=(patterns)
+      @whitelist = Array(patterns)
+      whitelist_subscribe
+    end
+    alias allowlist= whitelist=
+
     def intercept(pattern: "*", resource_type: nil)
       pattern = { urlPattern: pattern }
-      if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
-        pattern[:resourceType] = resource_type
-      end
+      pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
 
       @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
     end
 
     def authorize(user:, password:, type: :server, &block)
-      unless AUTHORIZE_TYPE.include?(type)
-        raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
-      end
-
-      if !block_given? && !@page.subscribed?("Fetch.requestPaused")
-        raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
-      end
+      raise ArgumentError, AUTHORIZE_TYPE_WRONG unless AUTHORIZE_TYPE.include?(type)
+      raise ArgumentError, AUTHORIZE_BLOCK_MISSING if !block_given? && !@page.subscribed?("Fetch.requestPaused")
 
       @authorized_ids ||= {}
       @authorized_ids[type] ||= []
@@ -116,6 +126,53 @@ module Ferrum
     end
 
     def subscribe
+      subscribe_request_will_be_sent
+      subscribe_response_received
+      subscribe_loading_finished
+      subscribe_loading_failed
+      subscribe_log_entry_added
+    end
+
+    def authorized_response(ids, request_id, username, password)
+      if ids.include?(request_id)
+        { response: "CancelAuth" }
+      elsif username && password
+        { response: "ProvideCredentials",
+          username: username,
+          password: password }
+      end
+    end
+
+    def select(request_id)
+      @traffic.select { |e| e.id == request_id }
+    end
+
+    def build_exchange(id)
+      Network::Exchange.new(@page, id).tap { |e| @traffic << e }
+    end
+
+    def emulate_network_conditions(offline: false, latency: 0,
+                                   download_throughput: -1, upload_throughput: -1,
+                                   connection_type: nil)
+      params = {
+        offline: offline, latency: latency,
+        downloadThroughput: download_throughput,
+        uploadThroughput: upload_throughput
+      }
+
+      params[:connectionType] = connection_type if connection_type && ALLOWED_CONNECTION_TYPE.include?(connection_type)
+
+      @page.command("Network.emulateNetworkConditions", **params)
+      true
+    end
+
+    def offline_mode
+      emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
+    end
+
+    private
+
+    def subscribe_request_will_be_sent
       @page.on("Network.requestWillBeSent") do |params|
         request = Network::Request.new(params)
 
@@ -140,25 +197,29 @@ module Ferrum
 
         exchange.request = request
 
-        if exchange.navigation_request?(@page.main_frame.id)
-          @exchange = exchange
-        end
+        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
       end
+    end
 
+    def subscribe_response_received
       @page.on("Network.responseReceived") do |params|
-        if exchange = select(params["requestId"]).last
+        exchange = select(params["requestId"]).last
+
+        if exchange
           response = Network::Response.new(@page, params)
           exchange.response = response
         end
       end
+    end
 
+    def subscribe_loading_finished
       @page.on("Network.loadingFinished") do |params|
         exchange = select(params["requestId"]).last
-        if exchange && exchange.response
-          exchange.response.body_size = params["encodedDataLength"]
-        end
+        exchange.response.body_size = params["encodedDataLength"] if exchange&.response
       end
+    end
 
+    def subscribe_loading_failed
       @page.on("Network.loadingFailed") do |params|
         exchange = select(params["requestId"]).last
         exchange.error ||= Network::Error.new
@@ -169,7 +230,9 @@ module Ferrum
         exchange.error.monotonic_time = params["timestamp"]
         exchange.error.canceled = params["canceled"]
       end
+    end
 
+    def subscribe_log_entry_added
       @page.on("Log.entryAdded") do |params|
         entry = params["entry"] || {}
         if entry["source"] == "network" && entry["level"] == "error"
@@ -184,24 +247,50 @@ module Ferrum
       end
     end
 
-    def authorized_response(ids, request_id, username, password)
-      if ids.include?(request_id)
-        { response: "CancelAuth" }
-      elsif username && password
-        { response: "ProvideCredentials",
-          username: username,
-          password: password }
-      else
-        { response: "CancelAuth" }
+    def blacklist_subscribe
+      return unless blacklist?
+      raise ArgumentError, "You can't use blacklist along with whitelist" if whitelist?
+
+      @blacklist_subscribe ||= begin
+        intercept
+
+        @page.on(:request) do |request|
+          if @blacklist.any? { |p| request.match?(p) }
+            request.abort
+          else
+            request.continue
+          end
+        end
+
+        true
       end
     end
 
-    def select(request_id)
-      @traffic.select { |e| e.id == request_id }
+    def whitelist_subscribe
+      return unless whitelist?
+      raise ArgumentError, "You can't use whitelist along with blacklist" if blacklist?
+
+      @whitelist_subscribe ||= begin
+        intercept
+
+        @page.on(:request) do |request|
+          if @whitelist.any? { |p| request.match?(p) }
+            request.continue
+          else
+            request.abort
+          end
+        end
+
+        true
+      end
     end
 
-    def build_exchange(id)
-      Network::Exchange.new(@page, id).tap { |e| @traffic << e }
+    def blacklist?
+      Array(@blacklist).any?
+    end
+
+    def whitelist?
+      Array(@whitelist).any?
     end
   end
 end

data/lib/ferrum/node.rb

@@ -2,15 +2,16 @@
 
 module Ferrum
   class Node
-    MOVING_WAIT = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
-    MOVING_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
+    MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
+    MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
 
     attr_reader :page, :target_id, :node_id, :description, :tag_name
 
     def initialize(frame, target_id, node_id, description)
       @page = frame.page
       @target_id = target_id
-      @node_id, @description = node_id, description
+      @node_id = node_id
+      @description = description
       @tag_name = description["nodeName"].downcase
     end
 
@@ -30,6 +31,27 @@ module Ferrum
       tap { page.command("DOM.focus", slowmoable: true, nodeId: node_id) }
     end
 
+    def focusable?
+      focus
+      true
+    rescue BrowserError => e
+      e.message == "Element is not focusable" ? false : raise
+    end
+
+    def wait_for_stop_moving(delay: MOVING_WAIT_DELAY, attempts: MOVING_WAIT_ATTEMPTS)
+      Utils::Attempt.with_retry(errors: NodeMovingError, max: attempts, wait: 0) do
+        previous, current = content_quads_with(delay: delay)
+        raise NodeMovingError.new(self, previous, current) if previous != current
+
+        current
+      end
+    end
+
+    def moving?(delay: MOVING_WAIT_DELAY)
+      previous, current = content_quads_with(delay: delay)
+      previous == current
+    end
+
     def blur
       tap { evaluate("this.blur()") }
     end
@@ -102,17 +124,52 @@ module Ferrum
     def property(name)
       evaluate("this['#{name}']")
     end
+    alias [] property
 
     def attribute(name)
       evaluate("this.getAttribute('#{name}')")
     end
 
+    def selected
+      function = <<~JS
+        function(element) {
+          if (element.nodeName.toLowerCase() !== 'select') {
+            throw new Error('Element is not a <select> element.');
+          }
+          return Array.from(element).filter(option => option.selected);
+        }
+      JS
+      page.evaluate_func(function, self, on: self)
+    end
+
+    def select(*values, by: :value)
+      tap do
+        function = <<~JS
+          function(element, values, by) {
+            if (element.nodeName.toLowerCase() !== 'select') {
+              throw new Error('Element is not a <select> element.');
+            }
+            const options = Array.from(element.options);
+            element.value = undefined;
+            for (const option of options) {
+              option.selected = values.some((value) => option[by] === value);
+              if (option.selected && !element.multiple) break;
+            }
+            element.dispatchEvent(new Event('input', { bubbles: true }));
+            element.dispatchEvent(new Event('change', { bubbles: true }));
+          }
+        JS
+        page.evaluate_func(function, self, values.flatten, by, on: self)
+      end
+    end
+
     def evaluate(expression)
       page.evaluate_on(node: self, expression: expression)
     end
 
     def ==(other)
       return false unless other.is_a?(Node)
+
       # We compare backendNodeId because once nodeId is sent to frontend backend
       # never returns same nodeId sending 0. In other words frontend is
       # responsible for keeping track of node ids.
@@ -124,44 +181,45 @@ module Ferrum
     end
 
     def find_position(x: nil, y: nil, position: :top)
-      prev = get_content_quads
-
-      # FIXME: Case when a few quads returned
-      points = Ferrum.with_attempts(errors: NodeIsMovingError, max: MOVING_ATTEMPTS, wait: 0) do
-        sleep(MOVING_WAIT)
-        current = get_content_quads
-
-        if current != prev
-          error = NodeIsMovingError.new(self, prev, current)
-          prev = current
-          raise(error)
-        end
-
-        current
-      end.map { |q| to_points(q) }.first
-
+      points = wait_for_stop_moving.map { |q| to_points(q) }.first
       get_position(points, x, y, position)
-    rescue Ferrum::BrowserError => e
-      return raise unless e.message&.include?("Could not compute content quads")
+    rescue CoordinatesNotFoundError
+      x, y = bounding_rect_coordinates
+      raise if x.zero? && y.zero?
 
-      find_position_via_js
+      [x, y]
+    end
+
+    # Returns a hash of the computed styles for the node
+    def computed_style
+      page
+        .command("CSS.getComputedStyleForNode", nodeId: node_id)["computedStyle"]
+        .each_with_object({}) { |style, memo| memo.merge!(style["name"] => style["value"]) }
     end
 
     private
 
-    def find_position_via_js
-      [
-        evaluate("this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2)"), # x
-        evaluate("this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)") # y
-      ]
+    def bounding_rect_coordinates
+      evaluate <<~JS
+        [this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2),
+         this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)]
+      JS
     end
 
-    def get_content_quads
+    def content_quads
       quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
-      raise "Node is either not visible or not an HTMLElement" if quads.size == 0
+      raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size.zero?
+
       quads
     end
 
+    def content_quads_with(delay: MOVING_WAIT_DELAY)
+      previous = content_quads
+      sleep(delay)
+      current = content_quads
+      [previous, current]
+    end
+
     def get_position(points, offset_x, offset_y, position)
       x = y = nil
 
@@ -170,28 +228,28 @@ module Ferrum
         x = point[:x] + offset_x.to_i
         y = point[:y] + offset_y.to_i
       else
-        x, y = points.inject([0, 0]) do |memo, point|
-          [memo[0] + point[:x],
-           memo[1] + point[:y]]
+        x, y = points.inject([0, 0]) do |memo, coordinate|
+          [memo[0] + coordinate[:x],
+           memo[1] + coordinate[:y]]
         end
 
-        x = x / 4
-        y = y / 4
+        x /= 4
+        y /= 4
       end
 
       if offset_x && offset_y && position == :center
-        x = x + offset_x.to_i
-        y = y + offset_y.to_i
+        x += offset_x.to_i
+        y += offset_y.to_i
       end
 
       [x, y]
     end
 
     def to_points(quad)
-      [{x: quad[0], y: quad[1]},
-       {x: quad[2], y: quad[3]},
-       {x: quad[4], y: quad[5]},
-       {x: quad[6], y: quad[7]}]
+      [{ x: quad[0], y: quad[1] },
+       { x: quad[2], y: quad[3] },
+       { x: quad[4], y: quad[5] },
+       { x: quad[6], y: quad[7] }]
     end
   end
 end

data/lib/ferrum/page/animation.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    module Animation
+      def playback_rate
+        command("Animation.getPlaybackRate")["playbackRate"]
+      end
+
+      def playback_rate=(value)
+        command("Animation.setPlaybackRate", playbackRate: value)
+      end
+    end
+  end
+end

data/lib/ferrum/page/frames.rb

@@ -11,35 +11,60 @@ module Ferrum
         @frames.values
       end
 
-      def frame_by(id: nil, name: nil)
+      def frame_by(id: nil, name: nil, execution_id: nil)
         if id
           @frames[id]
         elsif name
           frames.find { |f| f.name == name }
+        elsif execution_id
+          frames.find { |f| f.execution_id == execution_id }
         else
           raise ArgumentError
         end
       end
 
       def frames_subscribe
+        subscribe_frame_attached
+        subscribe_frame_started_loading
+        subscribe_frame_navigated
+        subscribe_frame_stopped_loading
+
+        subscribe_navigated_within_document
+
+        subscribe_request_will_be_sent
+
+        subscribe_execution_context_created
+        subscribe_execution_context_destroyed
+        subscribe_execution_contexts_cleared
+      end
+
+      private
+
+      def subscribe_frame_attached
         on("Page.frameAttached") do |params|
           parent_frame_id, frame_id = params.values_at("parentFrameId", "frameId")
           @frames[frame_id] = Frame.new(frame_id, self, parent_frame_id)
         end
+      end
 
+      def subscribe_frame_started_loading
         on("Page.frameStartedLoading") do |params|
           frame = @frames[params["frameId"]]
           frame.state = :started_loading
           @event.reset
         end
+      end
 
+      def subscribe_frame_navigated
         on("Page.frameNavigated") do |params|
           frame_id, name = params["frame"]&.values_at("id", "name")
           frame = @frames[frame_id]
           frame.state = :navigated
           frame.name = name unless name.to_s.empty?
         end
+      end
 
+      def subscribe_frame_stopped_loading
         on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the
@@ -47,7 +72,7 @@ module Ferrum
           # node will change the id and all subsequent nodes have to change id too.
           if @main_frame.id == params["frameId"]
             @event.set if idling?
-            get_document_id
+            document_node_id
           end
 
           frame = @frames[params["frameId"]]
@@ -55,21 +80,25 @@ module Ferrum
 
           @event.set if idling?
         end
+      end
 
+      def subscribe_navigated_within_document
         on("Page.navigatedWithinDocument") do
           @event.set if idling?
         end
+      end
 
+      def subscribe_request_will_be_sent
         on("Network.requestWillBeSent") do |params|
-          if params["frameId"] == @main_frame.id
-            # Possible types:
-            # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
-            # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
-            # CSPViolationReport, Other
-            @event.reset if params["type"] == "Document"
-          end
+          # Possible types:
+          # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
+          # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
+          # CSPViolationReport, Other
+          @event.reset if params["frameId"] == @main_frame.id && params["type"] == "Document"
         end
+      end
 
+      def subscribe_execution_context_created
         on("Runtime.executionContextCreated") do |params|
           context_id = params.dig("context", "id")
           frame_id = params.dig("context", "auxData", "frameId")
@@ -83,25 +112,27 @@ module Ferrum
           end
 
           frame = @frames[frame_id] || Frame.new(frame_id, self)
-          frame.set_execution_id(context_id)
+          frame.execution_id = context_id
 
           @frames[frame_id] ||= frame
         end
+      end
 
+      def subscribe_execution_context_destroyed
         on("Runtime.executionContextDestroyed") do |params|
           execution_id = params["executionContextId"]
-          frame = frames.find { |f| f.execution_id?(execution_id) }
-          frame.reset_execution_id
+          frame = frame_by(execution_id: execution_id)
+          frame&.execution_id = nil
         end
+      end
 
+      def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
           @frames.delete_if { |_, f| !f.main? }
-          @main_frame.reset_execution_id
+          @main_frame.execution_id = nil
         end
       end
 
-      private
-
       def idling?
         @frames.all? { |_, f| f.state == :stopped_loading }
       end