ferrum 0.6.2 → 0.10.1

data/lib/ferrum/headers.rb

@@ -40,7 +40,7 @@ module Ferrum
 
     def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
       options = Hash.new
-      options[:userAgent] = user_agent if user_agent
+      options[:userAgent] = user_agent || @page.browser.default_user_agent
       options[:acceptLanguage] = accept_language if accept_language
       options[:platform] if platform
 

data/lib/ferrum/keyboard.rb

@@ -33,13 +33,13 @@ module Ferrum
     def down(key)
       key = normalize_keys(Array(key))
       type = key[:text] ? "keyDown" : "rawKeyDown"
-      @page.command("Input.dispatchKeyEvent", type: type, **key)
+      @page.command("Input.dispatchKeyEvent", slowmoable: true, type: type, **key)
       self
     end
 
     def up(key)
       key = normalize_keys(Array(key))
-      @page.command("Input.dispatchKeyEvent", type: "keyUp", **key)
+      @page.command("Input.dispatchKeyEvent", slowmoable: true, type: "keyUp", **key)
       self
     end
 
@@ -49,7 +49,7 @@ module Ferrum
       keys.each do |key|
         type = key[:text] ? "keyDown" : "rawKeyDown"
         @page.command("Input.dispatchKeyEvent", type: type, **key)
-        @page.command("Input.dispatchKeyEvent", type: "keyUp", **key)
+        @page.command("Input.dispatchKeyEvent", slowmoable: true, type: "keyUp", **key)
       end
 
       self

data/lib/ferrum/mouse.rb

@@ -32,10 +32,21 @@ module Ferrum
       tap { mouse_event(type: "mouseReleased", **options) }
     end
 
-    # FIXME: steps
     def move(x:, y:, steps: 1)
+      from_x, from_y = @x, @y
       @x, @y = x, y
-      @page.command("Input.dispatchMouseEvent", type: "mouseMoved", x: @x, y: @y)
+
+      steps.times do |i|
+        new_x = from_x + (@x - from_x) * ((i + 1) / steps.to_f)
+        new_y = from_y + (@y - from_y) * ((i + 1) / steps.to_f)
+
+        @page.command("Input.dispatchMouseEvent",
+                      slowmoable: true,
+                      type: "mouseMoved",
+                      x: new_x.to_i,
+                      y: new_y.to_i)
+      end
+
       self
     end
 
@@ -45,7 +56,7 @@ module Ferrum
       button = validate_button(button)
       options = { x: @x, y: @y, type: type, button: button, clickCount: count }
       options.merge!(modifiers: modifiers) if modifiers
-      @page.command("Input.dispatchMouseEvent", wait: wait, **options)
+      @page.command("Input.dispatchMouseEvent", wait: wait, slowmoable: true, **options)
     end
 
     def validate_button(button)

data/lib/ferrum/network.rb

@@ -3,6 +3,9 @@
 require "ferrum/network/exchange"
 require "ferrum/network/intercepted_request"
 require "ferrum/network/auth_request"
+require "ferrum/network/error"
+require "ferrum/network/request"
+require "ferrum/network/response"
 
 module Ferrum
   class Network
@@ -20,6 +23,31 @@ module Ferrum
       @exchange = nil
     end
 
+    def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
+      start = Ferrum.monotonic_time
+
+      until idle?(connections)
+        raise TimeoutError if Ferrum.timeout?(start, timeout)
+        sleep(duration)
+      end
+    end
+
+    def idle?(connections = 0)
+      pending_connections <= connections
+    end
+
+    def total_connections
+      @traffic.size
+    end
+
+    def finished_connections
+      @traffic.count(&:finished?)
+    end
+
+    def pending_connections
+      total_connections - finished_connections
+    end
+
     def request
       @exchange&.request
     end
@@ -55,19 +83,21 @@ module Ferrum
       @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
     end
 
-    def authorize(user:, password:, type: :server)
+    def authorize(user:, password:, type: :server, &block)
       unless AUTHORIZE_TYPE.include?(type)
         raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
       end
 
+      if !block_given? && !@page.subscribed?("Fetch.requestPaused")
+        raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
+      end
+
       @authorized_ids ||= {}
       @authorized_ids[type] ||= []
 
       intercept
 
-      @page.on(:request) do |request|
-        request.continue
-      end
+      @page.on(:request, &block)
 
       @page.on(:auth) do |request, index, total|
         if request.auth_challenge?(type)
@@ -87,38 +117,69 @@ module Ferrum
 
     def subscribe
       @page.on("Network.requestWillBeSent") do |params|
+        request = Network::Request.new(params)
+
+        # We can build exchange in two places, here on the event or when request
+        # is interrupted. So we have to be careful when to create new one. We
+        # create new exchange only if there's no with such id or there's but
+        # it's filled with request which means this one is new but has response
+        # for a redirect. So we assign response from the params to previous
+        # exchange and build new exchange to assign this request to it.
+        exchange = select(request.id).last
+        exchange = build_exchange(request.id) unless exchange&.blank?
+
         # On redirects Chrome doesn't change `requestId` and there's no
         # `Network.responseReceived` event for such request. If there's already
         # exchange object with this id then we got redirected and params has
         # `redirectResponse` key which contains the response.
-        if exchange = first_by(params["requestId"])
-          exchange.build_response(params)
+        if params["redirectResponse"]
+          previous_exchange = select(request.id)[-2]
+          response = Network::Response.new(@page, params)
+          previous_exchange.response = response
         end
 
-        exchange = Network::Exchange.new(@page, params)
-        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
-        @traffic << exchange
+        exchange.request = request
+
+        if exchange.navigation_request?(@page.main_frame.id)
+          @exchange = exchange
+        end
       end
 
       @page.on("Network.responseReceived") do |params|
-        if exchange = last_by(params["requestId"])
-          exchange.build_response(params)
+        if exchange = select(params["requestId"]).last
+          response = Network::Response.new(@page, params)
+          exchange.response = response
         end
       end
 
       @page.on("Network.loadingFinished") do |params|
-        exchange = last_by(params["requestId"])
+        exchange = select(params["requestId"]).last
         if exchange && exchange.response
           exchange.response.body_size = params["encodedDataLength"]
         end
       end
 
+      @page.on("Network.loadingFailed") do |params|
+        exchange = select(params["requestId"]).last
+        exchange.error ||= Network::Error.new
+
+        exchange.error.id = params["requestId"]
+        exchange.error.type = params["type"]
+        exchange.error.error_text = params["errorText"]
+        exchange.error.monotonic_time = params["timestamp"]
+        exchange.error.canceled = params["canceled"]
+      end
+
       @page.on("Log.entryAdded") do |params|
         entry = params["entry"] || {}
-        if entry["source"] == "network" &&
-            entry["level"] == "error" &&
-            exchange = last_by(entry["networkRequestId"])
-          exchange.build_error(entry)
+        if entry["source"] == "network" && entry["level"] == "error"
+          exchange = select(entry["networkRequestId"]).last
+          exchange.error ||= Network::Error.new
+
+          exchange.error.id = entry["networkRequestId"]
+          exchange.error.url = entry["url"]
+          exchange.error.description = entry["text"]
+          exchange.error.timestamp = entry["timestamp"]
         end
       end
     end
@@ -135,12 +196,12 @@ module Ferrum
       end
     end
 
-    def first_by(request_id)
-      @traffic.find { |e| e.request.id == request_id }
+    def select(request_id)
+      @traffic.select { |e| e.id == request_id }
     end
 
-    def last_by(request_id)
-      @traffic.select { |e| e.request.id == request_id }.last
+    def build_exchange(id)
+      Network::Exchange.new(@page, id).tap { |e| @traffic << e }
     end
   end
 end

data/lib/ferrum/network/error.rb

@@ -3,24 +3,17 @@
 module Ferrum
   class Network
     class Error
-      def initialize(data)
-        @data = data
-      end
-
-      def id
-        @data["networkRequestId"]
-      end
-
-      def url
-        @data["url"]
-      end
+      attr_writer :canceled
+      attr_reader :time, :timestamp
+      attr_accessor :id, :url, :type, :error_text, :monotonic_time, :description
 
-      def description
-        @data["text"]
+      def canceled?
+        @canceled
       end
 
-      def time
-        @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
+      def timestamp=(value)
+        @timestamp = value
+        @time = Time.strptime((value / 1000).to_s, "%s")
       end
     end
   end

data/lib/ferrum/network/exchange.rb

@@ -1,39 +1,37 @@
 # frozen_string_literal: true
 
-require "ferrum/network/error"
-require "ferrum/network/request"
-require "ferrum/network/response"
-
 module Ferrum
   class Network
     class Exchange
-      attr_reader :request, :response, :error
+      attr_reader :id
+      attr_accessor :intercepted_request
+      attr_accessor :request, :response, :error
 
-      def initialize(page, params)
-        @page = page
-        @response = @error = nil
-        build_request(params)
+      def initialize(page, id)
+        @page, @id = page, id
+        @intercepted_request = nil
+        @request = @response = @error = nil
       end
 
-      def build_request(params)
-        @request = Network::Request.new(params)
+      def navigation_request?(frame_id)
+        request.type?(:document) &&
+          request.frame_id == frame_id
       end
 
-      def build_response(params)
-        @response = Network::Response.new(@page, params)
+      def blank?
+        !request
       end
 
-      def build_error(params)
-        @error = Network::Error.new(params)
+      def blocked?
+        intercepted_request && intercepted_request.status?(:aborted)
       end
 
-      def navigation_request?(frame_id)
-        request.type?(:document) &&
-          request.frame_id == frame_id
+      def finished?
+        blocked? || response || error
       end
 
-      def blocked?
-        response.nil?
+      def pending?
+        !finished?
       end
 
       def to_a
@@ -41,7 +39,12 @@ module Ferrum
       end
 
       def inspect
-        %(#<#{self.class} @id=#{@id.inspect} @request=#{@request.inspect} @response=#{@response.inspect} @error=#{@error.inspect}>)
+        "#<#{self.class} "\
+        "@id=#{@id.inspect} "\
+        "@intercepted_request=#{@intercepted_request.inspect} "\
+        "@request=#{@request.inspect} "\
+        "@response=#{@response.inspect} "\
+        "@error=#{@error.inspect}>"
       end
     end
   end

data/lib/ferrum/network/intercepted_request.rb

@@ -5,14 +5,20 @@ require "base64"
 module Ferrum
   class Network
     class InterceptedRequest
-      attr_accessor :request_id, :frame_id, :resource_type
+      attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
 
       def initialize(page, params)
+        @status = nil
         @page, @params = page, params
         @request_id = params["requestId"]
         @frame_id = params["frameId"]
         @resource_type = params["resourceType"]
         @request = params["request"]
+        @network_id = params["networkId"]
+      end
+
+      def status?(value)
+        @status == value.to_sym
       end
 
       def navigation_request?
@@ -25,7 +31,7 @@ module Ferrum
 
       def respond(**options)
         has_body = options.has_key?(:body)
-        headers = has_body ? { "content-length" => options.fetch(:body, '').length } : {}
+        headers = has_body ? { "content-length" => options.fetch(:body, "").length } : {}
         headers = headers.merge(options.fetch(:responseHeaders, {}))
 
         options = {responseCode: 200}.merge(options)
@@ -33,17 +39,20 @@ module Ferrum
           requestId: request_id,
           responseHeaders: header_array(headers),
         })
-        options = options.merge(body: Base64.encode64(options.fetch(:body, '')).strip) if has_body
+        options = options.merge(body: Base64.strict_encode64(options.fetch(:body, ""))) if has_body
 
+        @status = :responded
         @page.command("Fetch.fulfillRequest", **options)
       end
 
       def continue(**options)
         options = options.merge(requestId: request_id)
+        @status = :continued
         @page.command("Fetch.continueRequest", **options)
       end
 
       def abort
+        @status = :aborted
         @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
       end
 

data/lib/ferrum/network/response.rb

@@ -34,6 +34,10 @@ module Ferrum
       def headers_size
         @response["encodedDataLength"]
       end
+      
+      def type
+        @params["type"]
+      end
 
       def content_type
         @content_type ||= headers.find { |k, _| k.downcase == "content-type" }&.last&.sub(/;.*\z/, "")

data/lib/ferrum/node.rb

@@ -2,6 +2,9 @@
 
 module Ferrum
   class Node
+    MOVING_WAIT = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
+    MOVING_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
+
     attr_reader :page, :target_id, :node_id, :description, :tag_name
 
     def initialize(frame, target_id, node_id, description)
@@ -24,7 +27,7 @@ module Ferrum
     end
 
     def focus
-      tap { page.command("DOM.focus", nodeId: node_id) }
+      tap { page.command("DOM.focus", slowmoable: true, nodeId: node_id) }
     end
 
     def blur
@@ -37,22 +40,23 @@ module Ferrum
 
     # mode: (:left | :right | :double)
     # keys: (:alt, (:ctrl | :control), (:meta | :command), :shift)
-    # offset: { :x, :y }
-    def click(mode: :left, keys: [], offset: {})
-      x, y = find_position(offset[:x], offset[:y])
+    # offset: { :x, :y, :position (:top | :center) }
+    def click(mode: :left, keys: [], offset: {}, delay: 0)
+      x, y = find_position(**offset)
       modifiers = page.keyboard.modifiers(keys)
 
       case mode
       when :right
         page.mouse.move(x: x, y: y)
         page.mouse.down(button: :right, modifiers: modifiers)
+        sleep(delay)
         page.mouse.up(button: :right, modifiers: modifiers)
       when :double
         page.mouse.move(x: x, y: y)
         page.mouse.down(modifiers: modifiers, count: 2)
         page.mouse.up(modifiers: modifiers, count: 2)
       when :left
-        page.mouse.click(x: x, y: y, modifiers: modifiers)
+        page.mouse.click(x: x, y: y, modifiers: modifiers, delay: delay)
       end
 
       self
@@ -63,7 +67,7 @@ module Ferrum
     end
 
     def select_file(value)
-      page.command("DOM.setFileInputFiles", nodeId: node_id, files: Array(value))
+      page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
     end
 
     def at_xpath(selector)
@@ -119,35 +123,75 @@ module Ferrum
       %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
     end
 
-    def find_position(offset_x = nil, offset_y = nil)
-      quads = get_content_quads
-      offset_x, offset_y = offset_x.to_i, offset_y.to_i
+    def find_position(x: nil, y: nil, position: :top)
+      prev = get_content_quads
+
+      # FIXME: Case when a few quads returned
+      points = Ferrum.with_attempts(errors: NodeIsMovingError, max: MOVING_ATTEMPTS, wait: 0) do
+        sleep(MOVING_WAIT)
+        current = get_content_quads
+
+        if current != prev
+          error = NodeIsMovingError.new(self, prev, current)
+          prev = current
+          raise(error)
+        end
+
+        current
+      end.map { |q| to_points(q) }.first
+
+      get_position(points, x, y, position)
+    rescue Ferrum::BrowserError => e
+      return raise unless e.message&.include?("Could not compute content quads")
+
+      find_position_via_js
+    end
+
+    private
+
+    def find_position_via_js
+      [
+        evaluate("this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2)"), # x
+        evaluate("this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)") # y
+      ]
+    end
 
-      if offset_x > 0 || offset_y > 0
-        point = quads.first
-        [point[:x] + offset_x, point[:y] + offset_y]
+    def get_content_quads
+      quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
+      raise "Node is either not visible or not an HTMLElement" if quads.size == 0
+      quads
+    end
+
+    def get_position(points, offset_x, offset_y, position)
+      x = y = nil
+
+      if offset_x && offset_y && position == :top
+        point = points.first
+        x = point[:x] + offset_x.to_i
+        y = point[:y] + offset_y.to_i
       else
-        x, y = quads.inject([0, 0]) do |memo, point|
+        x, y = points.inject([0, 0]) do |memo, point|
           [memo[0] + point[:x],
            memo[1] + point[:y]]
         end
-        [x / 4, y / 4]
+
+        x = x / 4
+        y = y / 4
       end
-    end
 
-    private
+      if offset_x && offset_y && position == :center
+        x = x + offset_x.to_i
+        y = y + offset_y.to_i
+      end
 
-    def get_content_quads
-      result = page.command("DOM.getContentQuads", nodeId: node_id)
-      raise "Node is either not visible or not an HTMLElement" if result["quads"].size == 0
+      [x, y]
+    end
 
-      # FIXME: Case when a few quads returned
-      result["quads"].map do |quad|
-        [{x: quad[0], y: quad[1]},
-         {x: quad[2], y: quad[3]},
-         {x: quad[4], y: quad[5]},
-         {x: quad[6], y: quad[7]}]
-      end.first
+    def to_points(quad)
+      [{x: quad[0], y: quad[1]},
+       {x: quad[2], y: quad[3]},
+       {x: quad[4], y: quad[5]},
+       {x: quad[6], y: quad[7]}]
     end
   end
 end