ferrum 0.8 → 0.11

data/lib/ferrum/network.rb

@@ -83,19 +83,21 @@ module Ferrum
       @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
     end
 
-    def authorize(user:, password:, type: :server)
+    def authorize(user:, password:, type: :server, &block)
       unless AUTHORIZE_TYPE.include?(type)
         raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
       end
 
+      if !block_given? && !@page.subscribed?("Fetch.requestPaused")
+        raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
+      end
+
       @authorized_ids ||= {}
       @authorized_ids[type] ||= []
 
       intercept
 
-      @page.on(:request) do |request|
-        request.continue
-      end
+      @page.on(:request, &block)
 
       @page.on(:auth) do |request, index, total|
         if request.auth_challenge?(type)
@@ -157,12 +159,27 @@ module Ferrum
         end
       end
 
+      @page.on("Network.loadingFailed") do |params|
+        exchange = select(params["requestId"]).last
+        exchange.error ||= Network::Error.new
+
+        exchange.error.id = params["requestId"]
+        exchange.error.type = params["type"]
+        exchange.error.error_text = params["errorText"]
+        exchange.error.monotonic_time = params["timestamp"]
+        exchange.error.canceled = params["canceled"]
+      end
+
       @page.on("Log.entryAdded") do |params|
         entry = params["entry"] || {}
         if entry["source"] == "network" && entry["level"] == "error"
           exchange = select(entry["networkRequestId"]).last
-          error = Network::Error.new(entry)
-          exchange.error = error
+          exchange.error ||= Network::Error.new
+
+          exchange.error.id = entry["networkRequestId"]
+          exchange.error.url = entry["url"]
+          exchange.error.description = entry["text"]
+          exchange.error.timestamp = entry["timestamp"]
         end
       end
     end

data/lib/ferrum/network/error.rb

@@ -3,24 +3,17 @@
 module Ferrum
   class Network
     class Error
-      def initialize(data)
-        @data = data
-      end
-
-      def id
-        @data["networkRequestId"]
-      end
-
-      def url
-        @data["url"]
-      end
+      attr_writer :canceled
+      attr_reader :time, :timestamp
+      attr_accessor :id, :url, :type, :error_text, :monotonic_time, :description
 
-      def description
-        @data["text"]
+      def canceled?
+        @canceled
       end
 
-      def time
-        @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
+      def timestamp=(value)
+        @timestamp = value
+        @time = Time.strptime((value / 1000).to_s, "%s")
       end
     end
   end

data/lib/ferrum/network/intercepted_request.rb

@@ -39,7 +39,7 @@ module Ferrum
           requestId: request_id,
           responseHeaders: header_array(headers),
         })
-        options = options.merge(body: Base64.encode64(options.fetch(:body, "")).strip) if has_body
+        options = options.merge(body: Base64.strict_encode64(options.fetch(:body, ""))) if has_body
 
         @status = :responded
         @page.command("Fetch.fulfillRequest", **options)

data/lib/ferrum/node.rb

@@ -2,6 +2,9 @@
 
 module Ferrum
   class Node
+    MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
+    MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
+
     attr_reader :page, :target_id, :node_id, :description, :tag_name
 
     def initialize(frame, target_id, node_id, description)
@@ -27,6 +30,26 @@ module Ferrum
       tap { page.command("DOM.focus", slowmoable: true, nodeId: node_id) }
     end
 
+    def focusable?
+      focus
+      true
+    rescue BrowserError => e
+      e.message == "Element is not focusable" ? false : raise
+    end
+
+    def wait_for_stop_moving(delay: MOVING_WAIT_DELAY, attempts: MOVING_WAIT_ATTEMPTS)
+      Ferrum.with_attempts(errors: NodeMovingError, max: attempts, wait: 0) do
+        previous, current = get_content_quads_with(delay: delay)
+        raise NodeMovingError.new(self, previous, current) if previous != current
+        current
+      end
+    end
+
+    def moving?(delay: MOVING_WAIT_DELAY)
+      previous, current = get_content_quads_with(delay: delay)
+      previous == current
+    end
+
     def blur
       tap { evaluate("this.blur()") }
     end
@@ -121,16 +144,45 @@ module Ferrum
     end
 
     def find_position(x: nil, y: nil, position: :top)
-      offset_x, offset_y = x, y
-      quads = get_content_quads
+      points = wait_for_stop_moving.map { |q| to_points(q) }.first
+      get_position(points, x, y, position)
+    rescue CoordinatesNotFoundError
+      x, y = get_bounding_rect_coordinates
+      raise if x == 0 && y == 0
+      [x, y]
+    end
+
+    private
+
+    def get_bounding_rect_coordinates
+      evaluate <<~JS
+        [this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2),
+         this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)]
+      JS
+    end
+
+    def get_content_quads
+      quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
+      raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size == 0
+      quads
+    end
+
+    def get_content_quads_with(delay: MOVING_WAIT_DELAY)
+      previous = get_content_quads
+      sleep(delay)
+      current = get_content_quads
+      [previous, current]
+    end
+
+    def get_position(points, offset_x, offset_y, position)
       x = y = nil
 
       if offset_x && offset_y && position == :top
-        point = quads.first
+        point = points.first
         x = point[:x] + offset_x.to_i
         y = point[:y] + offset_y.to_i
       else
-        x, y = quads.inject([0, 0]) do |memo, point|
+        x, y = points.inject([0, 0]) do |memo, point|
           [memo[0] + point[:x],
            memo[1] + point[:y]]
         end
@@ -147,19 +199,11 @@ module Ferrum
       [x, y]
     end
 
-    private
-
-    def get_content_quads
-      result = page.command("DOM.getContentQuads", nodeId: node_id)
-      raise "Node is either not visible or not an HTMLElement" if result["quads"].size == 0
-
-      # FIXME: Case when a few quads returned
-      result["quads"].map do |quad|
-        [{x: quad[0], y: quad[1]},
-         {x: quad[2], y: quad[3]},
-         {x: quad[4], y: quad[5]},
-         {x: quad[6], y: quad[7]}]
-      end.first
+    def to_points(quad)
+      [{x: quad[0], y: quad[1]},
+       {x: quad[2], y: quad[3]},
+       {x: quad[4], y: quad[5]},
+       {x: quad[6], y: quad[7]}]
     end
   end
 end

data/lib/ferrum/page.rb

@@ -9,6 +9,7 @@ require "ferrum/dialog"
 require "ferrum/network"
 require "ferrum/page/frames"
 require "ferrum/page/screenshot"
+require "ferrum/page/animation"
 require "ferrum/browser/client"
 
 module Ferrum
@@ -32,10 +33,10 @@ module Ferrum
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
                 current_url current_title url title body doctype set_content
-                execution_id evaluate evaluate_on evaluate_async execute
+                execution_id evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
 
-    include Frames, Screenshot
+    include Frames, Screenshot, Animation
 
     attr_accessor :referrer
     attr_reader :target_id, :browser,
@@ -44,7 +45,7 @@ module Ferrum
 
     def initialize(target_id, browser)
       @frames = {}
-      @main_frame = Frame.new(nil, self) 
+      @main_frame = Frame.new(nil, self)
       @target_id, @browser = target_id, browser
       @event = Event.new.tap(&:set)
 
@@ -65,7 +66,7 @@ module Ferrum
       @browser.timeout
     end
 
-    def goto(url = nil)
+    def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
       response = command("Page.navigate", wait: GOTO_WAIT, **options)
@@ -78,9 +79,12 @@ module Ferrum
       end
       response["frameId"]
     rescue TimeoutError
-      pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
-      raise StatusError.new(options[:url], pendings) unless pendings.empty?
+      if @browser.pending_connection_errors
+        pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
+        raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
+      end
     end
+    alias goto go_to
 
     def close
       @headers.clear
@@ -89,15 +93,12 @@ module Ferrum
     end
 
     def resize(width: nil, height: nil, fullscreen: false)
-      result = @browser.command("Browser.getWindowForTarget", targetId: @target_id)
-      @window_id, @bounds = result.values_at("windowId", "bounds")
-
       if fullscreen
         width, height = document_size
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "fullscreen" })
+        set_window_bounds(windowState: "fullscreen")
       else
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "normal" })
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { width: width, height: height, windowState: "normal" })
+        set_window_bounds(windowState: "normal")
+        set_window_bounds(width: width, height: height)
       end
 
       command("Emulation.setDeviceMetricsOverride", slowmoable: true,
@@ -108,6 +109,14 @@ module Ferrum
                                                     fitWindow: false)
     end
 
+    def position
+      @browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
+    end
+
+    def position=(left:, top:)
+      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: { left: left, top: top })
+    end
+
     def refresh
       command("Page.reload", wait: timeout, slowmoable: true)
     end
@@ -125,12 +134,26 @@ module Ferrum
       history_navigate(delta: 1)
     end
 
+    def wait_for_reload(sec = 1)
+      @event.reset if @event.set?
+      @event.wait(sec)
+      @event.set
+    end
+
     def bypass_csp(value = true)
       enabled = !!value
       command("Page.setBypassCSP", enabled: enabled)
       enabled
     end
 
+    def window_id
+      @browser.command("Browser.getWindowForTarget", targetId: @target_id)["windowId"]
+    end
+
+    def set_window_bounds(bounds = {})
+      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: bounds)
+    end
+
     def command(method, wait: 0, slowmoable: false, **params)
       iteration = @event.reset if wait > 0
       sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
@@ -173,6 +196,10 @@ module Ferrum
       end
     end
 
+    def subscribed?(event)
+      @client.subscribed?(event)
+    end
+
     private
 
     def subscribe
@@ -255,7 +282,7 @@ module Ferrum
       nil_or_relative = url.nil? || url.relative?
 
       if nil_or_relative && !@browser.base_url
-        raise "Set :base_url browser's option or use absolute url in `goto`, you passed: #{url_or_path}"
+        raise "Set :base_url browser's option or use absolute url in `go_to`, you passed: #{url_or_path}"
       end
 
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s

data/lib/ferrum/page/animation.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    module Animation
+      def playback_rate
+        command("Animation.getPlaybackRate")["playbackRate"]
+      end
+
+
+      def playback_rate=(value)
+        command("Animation.setPlaybackRate", playbackRate: value)
+      end
+    end
+  end
+end

data/lib/ferrum/page/frames.rb

@@ -71,17 +71,25 @@ module Ferrum
         end
 
         on("Runtime.executionContextCreated") do |params|
+          setting_up_main_frame = false
           context_id = params.dig("context", "id")
           frame_id = params.dig("context", "auxData", "frameId")
 
           unless @main_frame.id
-            @main_frame.id = frame_id
-            @frames[frame_id] = @main_frame
+            root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
+            if frame_id == root_frame
+              setting_up_main_frame = true
+              @main_frame.id = frame_id
+              @frames[frame_id] = @main_frame
+            end
           end
 
           frame = @frames[frame_id] || Frame.new(frame_id, self)
           frame.set_execution_id(context_id)
 
+          # Set event because `execution_id` might raise NoExecutionContextError
+          @event.set if setting_up_main_frame
+
           @frames[frame_id] ||= frame
         end
 

data/lib/ferrum/page/screenshot.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "ferrum/rbga"
+
 module Ferrum
   class Page
     module Screenshot
@@ -10,7 +12,7 @@ module Ferrum
         scale: 1.0
       }.freeze
 
-      PAPEP_FORMATS = {
+      PAPER_FORMATS = {
         letter:   { width:  8.50, height: 11.00 },
         legal:    { width:  8.50, height: 14.00 },
         tabloid:  { width: 11.00, height: 17.00 },
@@ -24,19 +26,33 @@ module Ferrum
         A6:       { width:  4.13, height:  5.83 },
       }.freeze
 
+      STREAM_CHUNK = 128 * 1024
+
       def screenshot(**opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
-        data = capture_screenshot(options, opts[:full])
+        data = capture_screenshot(options, opts[:full], opts[:background_color])
         return data if encoding == :base64
-        save_file(path, data)
+
+        bin = Base64.decode64(data)
+        save_file(path, bin)
       end
 
       def pdf(**opts)
         path, encoding = common_options(**opts)
-        options = pdf_options(**opts)
-        data = command("Page.printToPDF", **options).fetch("data")
-        return data if encoding == :base64
+        options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
+        handle = command("Page.printToPDF", **options).fetch("stream")
+
+        if path
+          stream_to_file(handle, path: path)
+        else
+          stream_to_memory(handle, encoding: encoding)
+        end
+      end
+
+      def mhtml(path: nil)
+        data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
+        return data if path.nil?
         save_file(path, data)
       end
 
@@ -56,9 +72,31 @@ module Ferrum
       private
 
       def save_file(path, data)
-        bin = Base64.decode64(data)
-        return bin unless path
-        File.open(path.to_s, "wb") { |f| f.write(bin) }
+        return data unless path
+        File.open(path.to_s, "wb") { |f| f.write(data) }
+      end
+
+      def stream_to_file(handle, path:)
+        File.open(path, "wb") { |f| stream_to(handle, f) }
+        true
+      end
+
+      def stream_to_memory(handle, encoding:)
+        data = String.new("") # Mutable string has << and compatible to File
+        stream_to(handle, data)
+        encoding == :base64 ? Base64.encode64(data) : data
+      end
+
+      def stream_to(handle, output)
+        loop do
+          result = command("IO.read", handle: handle, size: STREAM_CHUNK)
+
+          data_chunk = result["data"]
+          data_chunk = Base64.decode64(data_chunk) if result["base64Encoded"]
+          output << data_chunk
+
+          break if result["eof"]
+        end
       end
 
       def common_options(encoding: :base64, path: nil, **_)
@@ -76,7 +114,7 @@ module Ferrum
             raise ArgumentError, "Specify :format or :paper_width, :paper_height"
           end
 
-          dimension = PAPEP_FORMATS.fetch(format)
+          dimension = PAPER_FORMATS.fetch(format)
           options.merge!(paper_width: dimension[:width],
                          paper_height: dimension[:height])
         end
@@ -134,9 +172,11 @@ module Ferrum
         option.to_s.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.to_sym
       end
 
-      def capture_screenshot(options, full)
+      def capture_screenshot(options, full, background_color)
         maybe_resize_fullscreen(full) do
-          command("Page.captureScreenshot", **options)
+          with_background_color(background_color) do
+            command("Page.captureScreenshot", **options)
+          end
         end.fetch("data")
       end
 
@@ -150,6 +190,18 @@ module Ferrum
       ensure
         resize(width: width, height: height) if full
       end
+
+      def with_background_color(color)
+        if color
+          raise ArgumentError, "Accept Ferrum::RGBA class only" unless color.is_a?(RGBA)
+
+          command("Emulation.setDefaultBackgroundColorOverride", color: color.to_h)
+        end
+
+        yield
+      ensure
+        command("Emulation.setDefaultBackgroundColorOverride") if color
+      end
     end
   end
 end