ferrum 0.15 → 0.17

data/lib/ferrum/contexts.rb

@@ -4,6 +4,8 @@ require "ferrum/context"
 
 module Ferrum
   class Contexts
+    ALLOWED_TARGET_TYPES = %w[page iframe].freeze
+
     include Enumerable
 
     attr_reader :contexts
@@ -67,12 +69,19 @@ module Ferrum
 
     private
 
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
     def subscribe
       @client.on("Target.attachedToTarget") do |params|
         info, session_id = params.values_at("targetInfo", "sessionId")
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
+        unless @contexts[context_id]
+          context = Context.new(@client, self, context_id)
+          @contexts[context_id] = context
+          @default_context ||= context
+        end
+
         @contexts[context_id]&.add_target(session_id: session_id, params: info)
         if params["waitingForDebugger"]
           @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
@@ -81,15 +90,21 @@ module Ferrum
 
       @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
-        @contexts[context_id]&.add_target(params: info)
+
+        if info["type"] == "iframe" &&
+           (target = @contexts[context_id].find_target { |t| t.connected? && t.page.frame_by(id: info["targetId"]) })
+          @contexts[context_id]&.add_target(session_id: target.page.client.session_id, params: info)
+        else
+          @contexts[context_id]&.add_target(params: info)
+        end
       end
 
       @client.on("Target.targetInfoChanged") do |params|
         info = params["targetInfo"]
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id, target_id = info.values_at("browserContextId", "targetId")
         @contexts[context_id]&.update_target(target_id, info)
@@ -105,6 +120,7 @@ module Ferrum
         context&.delete_target(params["targetId"])
       end
     end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
 
     def discover
       @client.command("Target.setDiscoverTargets", discover: true)

data/lib/ferrum/cookies.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "yaml"
 require "ferrum/cookies/cookie"
 
 module Ferrum
@@ -169,6 +170,32 @@ module Ferrum
       true
     end
 
+    #
+    # Stores all cookies of current page in a file.
+    #
+    # @return [Integer]
+    #
+    # @example
+    #   browser.cookies.store # => Integer
+    #
+    def store(path = "cookies.yml")
+      File.write(path, map(&:to_h).to_yaml)
+    end
+
+    #
+    # Loads all cookies from the file and sets them for current page.
+    #
+    # @return [true]
+    #
+    # @example
+    #   browser.cookies.load # => true
+    #
+    def load(path = "cookies.yml")
+      cookies = YAML.load_file(path)
+      cookies.each { |c| set(c) }
+      true
+    end
+
     private
 
     def default_domain

data/lib/ferrum/errors.rb

@@ -78,6 +78,13 @@ module Ferrum
     end
   end
 
+  class InvalidScreenshotFormatError < Error
+    def initialize(format)
+      valid_formats = Page::Screenshot::SUPPORTED_SCREENSHOT_FORMAT.join(" | ")
+      super("Invalid value #{format} for option `:format` (#{valid_formats})")
+    end
+  end
+
   class BrowserError < Error
     attr_reader :response
 
@@ -99,8 +106,7 @@ module Ferrum
 
   class NoExecutionContextError < BrowserError
     def initialize(response = nil)
-      response ||= { "message" => "There's no context available" }
-      super(response)
+      super(response || { "message" => "There's no context available" })
     end
   end
 

data/lib/ferrum/frame/dom.rb

@@ -14,7 +14,7 @@
 # any updates, for example, the node may be destroyed without any notification.
 # This is a way to keep a reference to the Node, when you don't necessarily want
 # to keep track of it. One example would be linking to the node from performance
-# data (e.g. relayout root node). BackendNodeId may be either resolved to
+# data (e.g. re-layout root node). BackendNodeId may be either resolved to
 # inspected node (DOM.pushNodesByBackendIdsToFrontend) or described in more
 # details (DOM.describeNode).
 module Ferrum
@@ -92,7 +92,23 @@ module Ferrum
       #   browser.body # => '<html itemscope="" itemtype="http://schema.org/WebPage" lang="ru"><head>...
       #
       def body
-        evaluate("document.documentElement.outerHTML")
+        evaluate("document.documentElement?.outerHTML") || ""
+      end
+
+      #
+      # Returns the element in which the window is embedded.
+      #
+      # @return [Node, nil]
+      #   The element in which the window is embedded.
+      #
+      # @example
+      #   browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
+      #   frame = browser.frames.last
+      #   frame.frame_element # => [Node]
+      #   frame.parent.parent.parent.frame_element # => nil
+      #
+      def frame_element
+        evaluate("window.frameElement")
       end
 
       #

data/lib/ferrum/frame.rb

@@ -94,6 +94,23 @@ module Ferrum
       @parent_id.nil?
     end
 
+    #
+    # Returns the parent frame if this frame is nested in another one.
+    #
+    # @return [Frame, nil]
+    #
+    # @example
+    #   browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
+    #   frame = browser.frames.last
+    #   frame.url # => "https://www.openstreetmap.org/export/embed.html?bbox=-0.004017949104309083%2C51.47612752641776%2C0.00030577182769775396%2C51.478569861898606&layer=mapnik"
+    #   frame.parent.main? # => false
+    #   frame.parent.parent.main? # => false
+    #   frame.parent.parent.parent.main? # => true
+    #
+    def parent
+      @page.frame_by(id: @parent_id) if @parent_id
+    end
+
     #
     # Sets a content of a given frame.
     #

data/lib/ferrum/keyboard.rb

@@ -93,7 +93,6 @@ module Ferrum
 
     private
 
-    # TODO: Refactor it, and try to simplify complexity
     # rubocop:disable Metrics/PerceivedComplexity
     # rubocop:disable Metrics/CyclomaticComplexity
     def normalize_keys(keys, pressed_keys = [], memo = [])

data/lib/ferrum/mouse.rb

@@ -3,11 +3,36 @@
 module Ferrum
   class Mouse
     CLICK_WAIT = ENV.fetch("FERRUM_CLICK_WAIT", 0.1).to_f
-    VALID_BUTTONS = %w[none left middle right back forward].freeze
+    BUTTON_MASKS = {
+      "none" => 0,
+      "left" => 1,
+      "right" => 2,
+      "middle" => 4,
+      "back" => 8,
+      "forward" => 16
+    }.freeze
 
     def initialize(page)
       @page = page
       @x = @y = 0
+      @buttons = 0
+    end
+
+    #
+    # Scroll page by the given amount x, y.
+    #
+    # @param [Integer] x
+    #  The horizontal pixel value that you want to scroll by.
+    #
+    # @param [Integer] y
+    #   The vertical pixel value that you want to scroll by.
+    #
+    # @example
+    #   browser.go_to("https://www.google.com/search?q=Ruby+headless+driver+for+Capybara")
+    #   browser.mouse.scroll_by(0, 400)
+    #
+    def scroll_by(x, y)
+      tap { @page.execute("window.scrollBy(#{x}, #{y})") }
     end
 
     #
@@ -107,9 +132,9 @@ module Ferrum
     #
     # Mouse move to given x and y.
     #
-    # @param [Integer] x
+    # @param [Number] x
     #
-    # @param [Integer] y
+    # @param [Number] y
     #
     # @param [Integer] steps
     #   Sends intermediate mousemove events.
@@ -129,8 +154,9 @@ module Ferrum
         @page.command("Input.dispatchMouseEvent",
                       slowmoable: true,
                       type: "mouseMoved",
-                      x: new_x.to_i,
-                      y: new_y.to_i)
+                      x: new_x,
+                      y: new_y,
+                      buttons: @buttons)
       end
 
       self
@@ -140,16 +166,26 @@ module Ferrum
 
     def mouse_event(type:, button: :left, count: 1, modifiers: nil, wait: 0)
       button = validate_button(button)
-      options = { x: @x, y: @y, type: type, button: button, clickCount: count }
+      register_event_button(type, button)
+      options = { x: @x, y: @y, type: type, button: button, buttons: @buttons, clickCount: count }
       options.merge!(modifiers: modifiers) if modifiers
       @page.command("Input.dispatchMouseEvent", wait: wait, slowmoable: true, **options)
     end
 
     def validate_button(button)
       button = button.to_s
-      raise "Invalid button: #{button}" unless VALID_BUTTONS.include?(button)
+      raise "Invalid button: #{button}" unless BUTTON_MASKS.key?(button)
 
       button
     end
+
+    def register_event_button(type, button)
+      case type
+      when "mousePressed"
+        @buttons |= BUTTON_MASKS[button]
+      when "mouseReleased"
+        @buttons &= ~BUTTON_MASKS[button]
+      end
+    end
   end
 end

data/lib/ferrum/network/exchange.rb

@@ -28,6 +28,15 @@ module Ferrum
       # @return [Error, nil]
       attr_accessor :error
 
+      # Determines if the network exchange is unknown due to
+      # a lost of its context
+      #
+      # @return Boolean
+      attr_accessor :unknown
+
+      # @api private
+      attr_accessor :request_extra_info
+
       #
       # Initializes the network exchange.
       #
@@ -40,6 +49,8 @@ module Ferrum
         @page = page
         @intercepted_request = nil
         @request = @response = @error = nil
+        @request_extra_info = nil
+        @unknown = false
       end
 
       #
@@ -54,6 +65,15 @@ module Ferrum
         request&.type?(:document) && request&.frame_id == frame_id
       end
 
+      #
+      # The loader ID of the request.
+      #
+      # @return [String, nil]
+      #
+      def loader_id
+        request&.loader_id
+      end
+
       #
       # Determines if the network exchange has a request.
       #
@@ -74,12 +94,12 @@ module Ferrum
 
       #
       # Determines if the request was blocked, a response was returned, or if an
-      # error occurred.
+      # error occurred or the exchange is unknown and cannot be inferred.
       #
       # @return [Boolean]
       #
       def finished?
-        blocked? || response&.loaded? || !error.nil? || ping?
+        blocked? || response&.loaded? || !error.nil? || ping? || blob? || unknown
       end
 
       #
@@ -127,6 +147,15 @@ module Ferrum
         !!request&.ping?
       end
 
+      #
+      # Determines if the exchange is blob.
+      #
+      # @return [Boolean]
+      #
+      def blob?
+        !!url&.start_with?("blob:")
+      end
+
       #
       # Returns request's URL.
       #
@@ -156,7 +185,8 @@ module Ferrum
           "@intercepted_request=#{@intercepted_request.inspect} " \
           "@request=#{@request.inspect} " \
           "@response=#{@response.inspect} " \
-          "@error=#{@error.inspect}>"
+          "@error=#{@error.inspect}> " \
+          "@unknown=#{@unknown.inspect}>"
       end
     end
   end

data/lib/ferrum/network/request.rb

@@ -71,6 +71,15 @@ module Ferrum
         @params["frameId"]
       end
 
+      #
+      # The loader ID of the request.
+      #
+      # @return [String]
+      #
+      def loader_id
+        @params["loaderId"]
+      end
+
       #
       # The request timestamp.
       #

data/lib/ferrum/network.rb

@@ -41,7 +41,7 @@ module Ferrum
     end
 
     #
-    # Waits for network idle or raises {Ferrum::TimeoutError} error.
+    # Waits for network idle.
     #
     # @param [Integer] connections
     #   how many connections are allowed for network to be idling,
@@ -52,21 +52,33 @@ module Ferrum
     # @param [Float] timeout
     #   During what time we try to check idle.
     #
-    # @raise [Ferrum::TimeoutError]
+    # @return [Boolean]
     #
     # @example
     #   browser.go_to("https://example.com/")
     #   browser.at_xpath("//a[text() = 'No UI changes button']").click
-    #   browser.network.wait_for_idle
+    #   browser.network.wait_for_idle # => false
     #
     def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.timeout)
       start = Utils::ElapsedTime.monotonic_time
 
       until idle?(connections)
-        raise TimeoutError if Utils::ElapsedTime.timeout?(start, timeout)
+        return false if Utils::ElapsedTime.timeout?(start, timeout)
 
         sleep(duration)
       end
+
+      true
+    end
+
+    #
+    # Waits for network idle or raises {Ferrum::TimeoutError} error.
+    # Accepts same arguments as `wait_for_idle`.
+    #
+    # @raise [Ferrum::TimeoutError]
+    def wait_for_idle!(...)
+      result = wait_for_idle(...)
+      raise TimeoutError unless result
     end
 
     def idle?(connections = 0)
@@ -88,7 +100,7 @@ module Ferrum
     #
     # Page request of the main frame.
     #
-    # @return [Request]
+    # @return [Request, nil]
     #
     # @example
     #   browser.go_to("https://github.com/")
@@ -356,29 +368,38 @@ module Ferrum
       @page.on("Network.requestWillBeSent") do |params|
         request = Network::Request.new(params)
 
-        # We can build exchange in two places, here on the event or when request
-        # is interrupted. So we have to be careful when to create new one. We
-        # create new exchange only if there's no with such id or there's but
-        # it's filled with request which means this one is new but has response
-        # for a redirect. So we assign response from the params to previous
-        # exchange and build new exchange to assign this request to it.
-        exchange = select(request.id).last
-        exchange = build_exchange(request.id) unless exchange&.blank?
-
         # On redirects Chrome doesn't change `requestId` and there's no
         # `Network.responseReceived` event for such request. If there's already
         # exchange object with this id then we got redirected and params has
         # `redirectResponse` key which contains the response.
-        if params["redirectResponse"]
-          previous_exchange = select(request.id)[-2]
+        if params["redirectResponse"] && (previous_exchange = select(request.id).last)
           response = Network::Response.new(@page, params)
           response.loaded = true
           previous_exchange.response = response
         end
 
+        # We can build exchange in two places, here on the event or when request
+        # is interrupted. So we have to be careful when to create new one. We
+        # create new exchange only if there's no with such id or there's, but
+        # it's filled with request which means this one is new but has response
+        # for a redirect. So we assign response from the params to previous
+        # exchange and build new exchange to assign this request to it.
+        exchange = select(request.id).last
+        exchange = build_exchange(request.id) if exchange.nil? || !exchange.blank?
+        request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame.id)
+          @exchange = exchange
+          classify_pending_exchanges(exchange.loader_id)
+        end
+      end
+
+      @page.on("Network.requestWillBeSentExtraInfo") do |params|
+        exchange = select(params["requestId"]).last
+        exchange ||= build_exchange(params["requestId"])
+        exchange.request_extra_info = params
+        exchange.request&.headers&.merge!(params["headers"])
       end
     end
 
@@ -397,6 +418,8 @@ module Ferrum
         exchange = select(params["requestId"]).last
         next unless exchange
 
+        exchange.unknown = false
+
         if (response = exchange.response)
           response.loaded = true
           response.body_size = params["encodedDataLength"]
@@ -479,5 +502,17 @@ module Ferrum
     def whitelist?
       Array(@whitelist).any?
     end
+
+    # When the main frame navigates Chrome doesn't send `Network.loadingFailed`
+    # for pending async requests. Therefore, we mark pending connections as unknown since
+    # they are not relevant to the current navigation.
+    def classify_pending_exchanges(new_loader_id)
+      @traffic.each do |exchange|
+        break if exchange.loader_id == new_loader_id
+        next unless exchange.pending?
+
+        exchange.unknown = true
+      end
+    end
   end
 end

data/lib/ferrum/node.rb

@@ -76,6 +76,7 @@ module Ferrum
       when :double
         page.mouse.move(x: x, y: y)
         page.mouse.down(modifiers: modifiers, count: 2)
+        sleep(delay)
         page.mouse.up(modifiers: modifiers, count: 2)
       when :left
         page.mouse.click(x: x, y: y, modifiers: modifiers, delay: delay)
@@ -217,6 +218,17 @@ module Ferrum
         .each_with_object({}) { |style, memo| memo.merge!(style["name"] => style["value"]) }
     end
 
+    def remove
+      page.command("DOM.removeNode", nodeId: node_id)
+    end
+
+    def exists?
+      page.command("DOM.resolveNode", nodeId: node_id)
+      true
+    rescue Ferrum::NodeNotFoundError
+      false
+    end
+
     private
 
     def bounding_rect_coordinates

data/lib/ferrum/page/frames.rb

@@ -36,7 +36,7 @@ module Ferrum
       end
 
       #
-      # Find frame by given options.
+      # Find a frame by given params.
       #
       # @param [String] id
       #   Unique frame's id that page provides.
@@ -60,8 +60,6 @@ module Ferrum
           frames.find { |f| f.name == name }
         elsif execution_id
           frames.find { |f| f.execution_id == execution_id }
-        else
-          raise ArgumentError
         end
       end
 
@@ -126,11 +124,11 @@ module Ferrum
         on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the
-          # tree and we save it in a variable because if we call that again root
+          # tree, and we save it in a variable because if we call that again root
           # node will change the id and all subsequent nodes have to change id too.
           if @main_frame.id == params["frameId"]
             @event.set if idling?
-            document_node_id
+            document_node_id(async: true)
           end
 
           frame = @frames[params["frameId"]]
@@ -179,12 +177,16 @@ module Ferrum
           execution_id = params["executionContextId"]
           frame = frame_by(execution_id: execution_id)
           frame&.execution_id = nil
+          frame&.state = :stopped_loading
         end
       end
 
       def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
-          @frames.each_value { |f| f.execution_id = nil }
+          @frames.each_value do |f|
+            f.execution_id = nil
+            f.state = :stopped_loading
+          end
         end
       end
 

data/lib/ferrum/page/screencast.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    module Screencast
+      # Starts sending frames to record screencast to the given block.
+      #
+      # @param [Hash{Symbol => Object}] opts
+      #
+      # @option opts [:jpeg, :png] :format
+      #   The format the image should be returned in.
+      #
+      # @option opts [Integer] :quality
+      #   The image quality. **Note:** 0-100 works for JPEG only.
+      #
+      # @option opts [Integer] :max_width
+      #   Maximum screencast frame width.
+      #
+      # @option opts [Integer] :max_height
+      #   Maximum screencast frame height.
+      #
+      # @option opts [Integer] :every_nth_frame
+      #   Send every n-th frame.
+      #
+      # @yield [data, metadata, session_id]
+      #   The given block receives the screencast frame along with metadata
+      #   about the frame and the screencast session ID.
+      #
+      # @yieldparam data [String]
+      #   Base64-encoded compressed image.
+      #
+      # @yieldparam metadata [Hash{String => Object}]
+      #   Screencast frame metadata.
+      #
+      # @option metadata [Integer] 'offsetTop'
+      #   Top offset in DIP.
+      #
+      # @option metadata [Integer] 'pageScaleFactor'
+      #   Page scale factor.
+      #
+      # @option metadata [Integer] 'deviceWidth'
+      #   Device screen width in DIP.
+      #
+      # @option metadata [Integer] 'deviceHeight'
+      #   Device screen height in DIP.
+      #
+      # @option metadata [Integer] 'scrollOffsetX'
+      #   Position of horizontal scroll in CSS pixels.
+      #
+      # @option metadata [Integer] 'scrollOffsetY'
+      #   Position of vertical scroll in CSS pixels.
+      #
+      # @option metadata [Float] 'timestamp'
+      #   (optional) Frame swap timestamp in seconds since Unix epoch.
+      #
+      # @yieldparam session_id [Integer]
+      #   Frame number.
+      #
+      # @example
+      #   require "base64"
+      #
+      #   page.go_to("https://apple.com/ipad")
+      #
+      #   page.start_screencast(format: :jpeg, quality: 75) do |data, metadata|
+      #     timestamp = (metadata['timestamp'] * 1000).to_i
+      #     File.binwrite("image_#{timestamp}.jpg", Base64.decode64(data))
+      #   end
+      #
+      #   sleep 10
+      #
+      #   page.stop_screencast
+      #
+      def start_screencast(**opts)
+        options = opts.transform_keys { START_SCREENCAST_KEY_CONV.fetch(_1, _1) }
+        response = command("Page.startScreencast", **options)
+
+        if (error_text = response["errorText"]) # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
+          raise "Starting screencast failed (#{error_text})"
+        end
+
+        on("Page.screencastFrame") do |params|
+          data, metadata, session_id = params.values_at("data", "metadata", "sessionId")
+
+          command("Page.screencastFrameAck", sessionId: session_id)
+
+          yield data, metadata, session_id
+        end
+      end
+
+      # Stops sending frames.
+      def stop_screencast
+        command("Page.stopScreencast")
+      end
+
+      START_SCREENCAST_KEY_CONV = {
+        max_width: :maxWidth,
+        max_height: :maxHeight,
+        every_nth_frame: :everyNthFrame
+      }.freeze
+    end
+  end
+end