ferrum 0.16 → 0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa8c09869c0473b537bed621b05616d1b062bc8c639df527b5bb60cb3722e854
-  data.tar.gz: f7277b98ba6ca90e33b92573c88926042296b3e33bb9acb54b2a6946294dba1d
+  metadata.gz: f586cea5a9987e2a53bfb860fb503d36fc9e2f4e16b7288dafe68b2e62f20ede
+  data.tar.gz: a33d17e091b0d7b506f0934dd19f056ab15c72891659cc227f03bccad8d90cdd
 SHA512:
-  metadata.gz: fd16b7e9d221cfc0c51f35af1362b486cb877120abfb61a515eed7f0b11733bdbdb2b54e089ef12599fa5b040fa85dd56315ac756353463d417482781be40644
-  data.tar.gz: 2f1db2e77922c37efbd83a4aada4df7870afa676e6777fe7e437646c5c05f3d9eeece819c6ff001d53ace2e7d20edf9d1316d7ebcbb843fa695c3697069555f8
+  metadata.gz: 031a4dcb48ba0d22c1070ab8e905881f07369e7c19b61f03ef747d4ff3813be097a8f8bc65f692f4ed186687ba1e4e83c9de4482939a963b2541364c0d745b16
+  data.tar.gz: 207975dde463249331095cada046454110ee80d8a4a42fdc7f51898ad514e0aa5395946e037abd5c5599a7fa2d570c279925555822d432c4470c6e955f37b85b

data/README.md

@@ -34,6 +34,7 @@ based on Ferrum and Mechanize.
 * [Navigation](https://github.com/rubycdp/ferrum#navigation)
 * [Finders](https://github.com/rubycdp/ferrum#finders)
 * [Screenshots](https://github.com/rubycdp/ferrum#screenshots)
+* [Screencast](https://github.com/rubycdp/ferrum#screencast)
 * [Network](https://github.com/rubycdp/ferrum#network)
 * [Downloads](https://github.com/rubycdp/ferrum#downloads)
 * [Proxy](https://github.com/rubycdp/ferrum#proxy)
@@ -136,7 +137,7 @@ browser.quit
 In docker as root you must pass the no-sandbox browser option:
 
 ```ruby
-Ferrum::Browser.new(browser_options: { 'no-sandbox': nil })
+Ferrum::Browser.new(browser_options: { "no-sandbox": nil })
 ```
 
 It has also been reported that the Chrome process repeatedly crashes when running inside a Docker container on an M1 Mac preventing Ferrum from working. Ferrum should work as expected when deployed to a Docker container on a non-M1 Mac.
@@ -150,8 +151,8 @@ Ferrum::Browser.new(options)
 ```
 
 * options `Hash`
-  * `:headless` (String | Boolean) - Set browser as headless or not, `true` by default. You can set `"new"` to support
-      [new headless mode](https://developer.chrome.com/articles/new-headless/).
+  * `:headless` (Boolean) - Set browser as headless or not, `true` by default.
+  * `:incognito` (Boolean) - Create an incognito profile for the browser startup window, `true` by default.
   * `:xvfb` (Boolean) - Run browser in a virtual framebuffer, `false` by default.
   * `:flatten` (Boolean) - Use one websocket connection to the browser and all the pages in flatten mode.
   * `:window_size` (Array) - The dimensions of the browser window in which to
@@ -416,7 +417,7 @@ page.screenshot(path: "google.jpg") # => 30902
 # Save to Base64 the whole page not only viewport and reduce quality
 page.screenshot(full: true, quality: 60, encoding: :base64) # "iVBORw0KGgoAAAANSUhEUgAABAAAAAMACAYAAAC6uhUNAAAAAXNSR0IArs4c6Q...
 # Save on the disk with the selected element in PNG
-page.screenshot(path: "google.png", selector: 'textarea') # => 11340
+page.screenshot(path: "google.png", selector: "textarea") # => 11340
 # Save to Base64 with an area of the page in PNG
 page.screenshot(path: "google.png", area: { x: 0, y: 0, width: 400, height: 300 }) # => 54239
 # Save with specific background color
@@ -458,6 +459,58 @@ page.go_to("https://google.com/")
 page.mhtml(path: "google.mhtml") # => 87742
 ```
 
+## Screencast
+
+#### start_screencast(\*\*options) { |data, metadata, session_id| ... }
+
+Starts sending frames to record screencast to the given block.
+
+* options `Hash`
+  * :format `Symbol` `:jpeg` | `:png` The format the image should be returned in.
+  * :quality `Integer` The image quality. **Note:** 0-100 works for JPEG only.
+  * :max_width `Integer` Maximum screencast frame width.
+  * :max_height `Integer` Maximum screencast frame height.
+  * :every_nth_frame `Integer` Send every n-th frame.
+
+* Block inputs:
+  * data `String` Base64-encoded compressed image.
+  * metadata `Hash` Screencast frame metadata.
+    * "offsetTop" `Integer` Top offset in DIP.
+    * "pageScaleFactor" `Integer` Page scale factor.
+    * "deviceWidth" `Integer` Device screen width in DIP.
+    * "deviceHeight" `Integer` Device screen height in DIP.
+    * "scrollOffsetX" `Integer` Position of horizontal scroll in CSS pixels.
+    * "scrollOffsetY" `Integer` Position of vertical scroll in CSS pixels.
+    * "timestamp" `Float` (optional) Frame swap timestamp in seconds since Unix epoch.
+  * session_id `Integer` Frame number.
+
+```ruby
+require "base64"
+
+page.go_to("https://apple.com/ipad")
+
+page.start_screencast(format: :jpeg, quality: 75) do |data, metadata|
+  timestamp = (metadata["timestamp"] * 1000).to_i
+  File.binwrite("image_#{timestamp}.jpg", Base64.decode64(data))
+end
+
+sleep 10
+
+page.stop_screencast
+```
+
+> ### 📝 NOTE
+>
+> Chrome only sends new frames while page content is changing. For example, if
+> there is an animation or a video on the page, Chrome sends frames at the rate
+> requested. On the other hand, if the page is nothing but a wall of static text,
+> Chrome sends frames while the page renders. Once Chrome has finished rendering
+> the page, it sends no more frames until something changes (e.g., navigating to
+> another location).
+
+#### stop_screencast
+
+Stops sending frames.
 
 ## Network
 
@@ -1085,9 +1138,17 @@ Frame's unique id.
 
 Parent frame id if this one is nested in another one.
 
+#### parent : `Frame | nil`
+
+Parent frame if this one is nested in another one.
+
+#### frame_element : `Node | nil`
+
+Returns the element in which the window is embedded.
+
 #### execution_id : `Integer`
 
-Execution context id which is used by JS, each frame has it's own context in
+Execution context id which is used by JS, each frame has its own context in
 which JS evaluates.
 
 #### name : `String | nil`
@@ -1322,7 +1383,7 @@ Closes browser tabs opened by the `Browser` instance.
 
 ```ruby
 # connect to a long-running Chrome process
-browser = Ferrum::Browser.new(url: 'http://localhost:9222')
+browser = Ferrum::Browser.new(url: "http://localhost:9222")
 
 browser.go_to("https://github.com/")
 

data/lib/ferrum/browser/command.rb

@@ -39,10 +39,6 @@ module Ferrum
         !!options.xvfb
       end
 
-      def headless_new?
-        @flags["headless"] == "new"
-      end
-
       def to_a
         [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
       end

data/lib/ferrum/browser/options/base.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "singleton"
+require "open3"
 
 module Ferrum
   class Browser
@@ -12,6 +13,14 @@ module Ferrum
           instance
         end
 
+        # @return [String, nil]
+        def self.version
+          out, = Open3.capture2(instance.detect_path, "--version")
+          out.strip
+        rescue Errno::ENOENT
+          nil
+        end
+
         def to_h
           self.class::DEFAULT_OPTIONS
         end

data/lib/ferrum/browser/options/chrome.rb

@@ -38,8 +38,9 @@ module Ferrum
           "safebrowsing-disable-auto-update" => nil,
           "password-store" => "basic",
           "no-startup-window" => nil,
-          "remote-allow-origins" => "*"
-          # NOTE: --no-sandbox is not needed if you properly setup a user in the container.
+          "remote-allow-origins" => "*",
+          "disable-blink-features" => "AutomationControlled"
+          # NOTE: --no-sandbox is not needed if you properly set up a user in the container.
           # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
           # "no-sandbox" => nil,
         }.freeze
@@ -75,15 +76,10 @@ module Ferrum
         end
 
         def merge_default(flags, options)
-          defaults = case options.headless
-                     when false
-                       except("headless", "disable-gpu")
-                     when "new"
-                       except("headless").merge("headless" => "new")
-                     end
-
+          defaults = except("headless", "disable-gpu") if options.headless == false
           defaults ||= DEFAULT_OPTIONS
-          # On Windows, the --disable-gpu flag is a temporary work around for a few bugs.
+          defaults.delete("no-startup-window") if options.incognito == false
+          # On Windows, the --disable-gpu flag is a temporary workaround for a few bugs.
           # See https://bugs.chromium.org/p/chromium/issues/detail?id=737678 for more information.
           defaults = defaults.merge("disable-gpu" => nil) if Utils::Platform.windows?
           # Use Metal on Apple Silicon

data/lib/ferrum/browser/options.rb

@@ -14,7 +14,7 @@ module Ferrum
       attr_reader :window_size, :logger, :ws_max_receive_size,
                   :js_errors, :base_url, :slowmo, :pending_connection_errors,
                   :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
-                  :save_path, :proxy, :port, :host, :headless, :browser_options,
+                  :save_path, :proxy, :port, :host, :headless, :incognito, :browser_options,
                   :ignore_default_browser_options, :xvfb, :flatten
       attr_accessor :timeout, :default_user_agent
 
@@ -27,6 +27,7 @@ module Ferrum
         @window_size = @options.fetch(:window_size, WINDOW_SIZE)
         @js_errors = @options.fetch(:js_errors, false)
         @headless = @options.fetch(:headless, true)
+        @incognito = @options.fetch(:incognito, true)
         @flatten = @options.fetch(:flatten, true)
         @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)

data/lib/ferrum/browser/process.rb

@@ -178,7 +178,7 @@ module Ferrum
 
       def close_io(*ios)
         ios.each do |io|
-          io.close unless io.closed?
+          io.close if io && !io.closed?
         rescue IOError
           raise unless RUBY_ENGINE == "jruby"
         end

data/lib/ferrum/browser.rb

@@ -23,6 +23,7 @@ module Ferrum
                 headers cookies network downloads
                 mouse keyboard
                 screenshot pdf mhtml viewport_size device_pixel_ratio
+                start_screencast stop_screencast
                 frames frame_by main_frame
                 evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
@@ -44,6 +45,9 @@ module Ferrum
     # @option options [Boolean] :headless (true)
     #   Set browser as headless or not.
     #
+    # @option options [Boolean] :incognito (true)
+    #   Create an incognito profile for the browser startup window.
+    #
     # @option options [Boolean] :xvfb (false)
     #   Run browser in a virtual framebuffer.
     #
@@ -245,8 +249,22 @@ module Ferrum
       VersionInfo.new(command("Browser.getVersion"))
     end
 
-    def headless_new?
-      process&.command&.headless_new?
+    #
+    # Opens headless session in the browser devtools frontend.
+    #
+    # @return [void]
+    #
+    # @since 0.16
+    #
+    def debug(bind = nil)
+      ::Process.spawn(process.path, debug_url)
+
+      bind ||= binding
+      if bind.respond_to?(:pry)
+        Pry.start(bind)
+      else
+        bind.irb
+      end
     end
 
     private
@@ -266,5 +284,18 @@ module Ferrum
         raise
       end
     end
+
+    def debug_url
+      response = JSON.parse(Net::HTTP.get(URI(build_remote_debug_url(path: "/json"))))
+
+      devtools_frontend_path = response[0]&.[]("devtoolsFrontendUrl")
+      raise "Could not generate debug url for remote debugging session" unless devtools_frontend_path
+
+      build_remote_debug_url(path: devtools_frontend_path)
+    end
+
+    def build_remote_debug_url(path:)
+      "http://#{process.host}:#{process.port}#{path}"
+    end
   end
 end

data/lib/ferrum/client/subscriber.rb

@@ -8,7 +8,7 @@ module Ferrum
       def initialize
         @regular = Queue.new
         @priority = Queue.new
-        @on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
+        @on = Concurrent::Hash.new
 
         start
       end
@@ -22,7 +22,13 @@ module Ferrum
       end
 
       def on(event, &block)
+        @on[event] ||= Concurrent::Array.new
         @on[event] << block
+        @on[event].index(block)
+      end
+
+      def off(event, id)
+        @on[event].delete_at(id)
         true
       end
 
@@ -65,8 +71,8 @@ module Ferrum
         method, session_id, params = message.values_at("method", "sessionId", "params")
         event = SessionClient.event_name(method, session_id)
 
-        total = @on[event].size
-        @on[event].each_with_index do |block, index|
+        total = @on[event]&.size.to_i
+        @on[event]&.each_with_index do |block, index|
           # In case of multiple callbacks we provide current index and total
           block.call(params, index, total)
         end

data/lib/ferrum/client.rb

@@ -28,6 +28,10 @@ module Ferrum
       @client.on(event_name(event), &block)
     end
 
+    def off(event, id)
+      @client.off(event_name(event), id)
+    end
+
     def subscribed?(event)
       @client.subscribed?(event_name(event))
     end
@@ -101,6 +105,10 @@ module Ferrum
       @subscriber.on(event, &block)
     end
 
+    def off(event, id)
+      @subscriber.off(event, id)
+    end
+
     def subscribed?(event)
       @subscriber.subscribed?(event)
     end

data/lib/ferrum/context.rb

@@ -13,7 +13,7 @@ module Ferrum
       @client = client
       @contexts = contexts
       @targets = Concurrent::Map.new
-      @pendings = Concurrent::MVar.new
+      @pendings = Concurrent::Map.new
     end
 
     def default_target
@@ -25,11 +25,11 @@ module Ferrum
     end
 
     def pages
-      @targets.values.map(&:page)
+      @targets.values.reject(&:iframe?).map(&:page)
     end
 
     # When we call `page` method on target it triggers ruby to connect to given
-    # page by WebSocket, if there are many opened windows but we need only one
+    # page by WebSocket, if there are many opened windows, but we need only one
     # it makes more sense to get and connect to the needed one only which
     # usually is the last one.
     def windows(pos = nil, size = 1)
@@ -46,19 +46,27 @@ module Ferrum
     end
 
     def create_target
-      @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")
-      target = @pendings.take(@client.timeout)
-      raise NoSuchTargetError unless target.is_a?(Target)
+      target_id = @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")["targetId"]
 
-      target
+      new_pending = Concurrent::IVar.new
+      pending = @pendings.put_if_absent(target_id, new_pending) || new_pending
+      resolved = pending.value(@client.timeout)
+      raise NoSuchTargetError unless resolved
+
+      @pendings.delete(target_id)
+      @targets[target_id]
     end
 
     def add_target(params:, session_id: nil)
       new_target = Target.new(@client, session_id, params)
-      target = @targets.put_if_absent(new_target.id, new_target)
-      target ||= new_target # `put_if_absent` returns nil if added a new value or existing if there was one already
-      @pendings.put(target, @client.timeout) if @pendings.empty?
-      target
+      # `put_if_absent` returns nil if added a new value or existing if there was one already
+      target = @targets.put_if_absent(new_target.id, new_target) || new_target
+      @default_target ||= target
+
+      new_pending = Concurrent::IVar.new
+      pending = @pendings.put_if_absent(target.id, new_pending) || new_pending
+      pending.try_set(true)
+      true
     end
 
     def update_target(target_id, params)
@@ -69,6 +77,21 @@ module Ferrum
       @targets.delete(target_id)
     end
 
+    def attach_target(target_id)
+      target = @targets[target_id]
+      raise NoSuchTargetError unless target
+
+      session = @client.command("Target.attachToTarget", targetId: target_id, flatten: true)
+      target.session_id = session["sessionId"]
+      true
+    end
+
+    def find_target
+      @targets.each_value { |t| return t if yield(t) }
+
+      nil
+    end
+
     def close_targets_connection
       @targets.each_value do |target|
         next unless target.connected?

data/lib/ferrum/contexts.rb

@@ -4,6 +4,8 @@ require "ferrum/context"
 
 module Ferrum
   class Contexts
+    ALLOWED_TARGET_TYPES = %w[page iframe].freeze
+
     include Enumerable
 
     attr_reader :contexts
@@ -67,12 +69,19 @@ module Ferrum
 
     private
 
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
     def subscribe
       @client.on("Target.attachedToTarget") do |params|
         info, session_id = params.values_at("targetInfo", "sessionId")
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
+        unless @contexts[context_id]
+          context = Context.new(@client, self, context_id)
+          @contexts[context_id] = context
+          @default_context ||= context
+        end
+
         @contexts[context_id]&.add_target(session_id: session_id, params: info)
         if params["waitingForDebugger"]
           @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
@@ -81,15 +90,21 @@ module Ferrum
 
       @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
-        @contexts[context_id]&.add_target(params: info)
+
+        if info["type"] == "iframe" &&
+           (target = @contexts[context_id].find_target { |t| t.connected? && t.page.frame_by(id: info["targetId"]) })
+          @contexts[context_id]&.add_target(session_id: target.page.client.session_id, params: info)
+        else
+          @contexts[context_id]&.add_target(params: info)
+        end
       end
 
       @client.on("Target.targetInfoChanged") do |params|
         info = params["targetInfo"]
-        next unless info["type"] == "page"
+        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id, target_id = info.values_at("browserContextId", "targetId")
         @contexts[context_id]&.update_target(target_id, info)
@@ -105,6 +120,7 @@ module Ferrum
         context&.delete_target(params["targetId"])
       end
     end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
 
     def discover
       @client.command("Target.setDiscoverTargets", discover: true)

data/lib/ferrum/frame/dom.rb

@@ -14,7 +14,7 @@
 # any updates, for example, the node may be destroyed without any notification.
 # This is a way to keep a reference to the Node, when you don't necessarily want
 # to keep track of it. One example would be linking to the node from performance
-# data (e.g. relayout root node). BackendNodeId may be either resolved to
+# data (e.g. re-layout root node). BackendNodeId may be either resolved to
 # inspected node (DOM.pushNodesByBackendIdsToFrontend) or described in more
 # details (DOM.describeNode).
 module Ferrum
@@ -92,7 +92,23 @@ module Ferrum
       #   browser.body # => '<html itemscope="" itemtype="http://schema.org/WebPage" lang="ru"><head>...
       #
       def body
-        evaluate("document.documentElement.outerHTML")
+        evaluate("document.documentElement?.outerHTML") || ""
+      end
+
+      #
+      # Returns the element in which the window is embedded.
+      #
+      # @return [Node, nil]
+      #   The element in which the window is embedded.
+      #
+      # @example
+      #   browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
+      #   frame = browser.frames.last
+      #   frame.frame_element # => [Node]
+      #   frame.parent.parent.parent.frame_element # => nil
+      #
+      def frame_element
+        evaluate("window.frameElement")
       end
 
       #

data/lib/ferrum/frame.rb

@@ -94,6 +94,23 @@ module Ferrum
       @parent_id.nil?
     end
 
+    #
+    # Returns the parent frame if this frame is nested in another one.
+    #
+    # @return [Frame, nil]
+    #
+    # @example
+    #   browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
+    #   frame = browser.frames.last
+    #   frame.url # => "https://www.openstreetmap.org/export/embed.html?bbox=-0.004017949104309083%2C51.47612752641776%2C0.00030577182769775396%2C51.478569861898606&layer=mapnik"
+    #   frame.parent.main? # => false
+    #   frame.parent.parent.main? # => false
+    #   frame.parent.parent.parent.main? # => true
+    #
+    def parent
+      @page.frame_by(id: @parent_id) if @parent_id
+    end
+
     #
     # Sets a content of a given frame.
     #

data/lib/ferrum/keyboard.rb

@@ -93,7 +93,6 @@ module Ferrum
 
     private
 
-    # TODO: Refactor it, and try to simplify complexity
     # rubocop:disable Metrics/PerceivedComplexity
     # rubocop:disable Metrics/CyclomaticComplexity
     def normalize_keys(keys, pressed_keys = [], memo = [])

data/lib/ferrum/mouse.rb

@@ -3,11 +3,36 @@
 module Ferrum
   class Mouse
     CLICK_WAIT = ENV.fetch("FERRUM_CLICK_WAIT", 0.1).to_f
-    VALID_BUTTONS = %w[none left middle right back forward].freeze
+    BUTTON_MASKS = {
+      "none" => 0,
+      "left" => 1,
+      "right" => 2,
+      "middle" => 4,
+      "back" => 8,
+      "forward" => 16
+    }.freeze
 
     def initialize(page)
       @page = page
       @x = @y = 0
+      @buttons = 0
+    end
+
+    #
+    # Scroll page by the given amount x, y.
+    #
+    # @param [Integer] x
+    #  The horizontal pixel value that you want to scroll by.
+    #
+    # @param [Integer] y
+    #   The vertical pixel value that you want to scroll by.
+    #
+    # @example
+    #   browser.go_to("https://www.google.com/search?q=Ruby+headless+driver+for+Capybara")
+    #   browser.mouse.scroll_by(0, 400)
+    #
+    def scroll_by(x, y)
+      tap { @page.execute("window.scrollBy(#{x}, #{y})") }
     end
 
     #
@@ -107,9 +132,9 @@ module Ferrum
     #
     # Mouse move to given x and y.
     #
-    # @param [Integer] x
+    # @param [Number] x
     #
-    # @param [Integer] y
+    # @param [Number] y
     #
     # @param [Integer] steps
     #   Sends intermediate mousemove events.
@@ -129,8 +154,9 @@ module Ferrum
         @page.command("Input.dispatchMouseEvent",
                       slowmoable: true,
                       type: "mouseMoved",
-                      x: new_x.to_i,
-                      y: new_y.to_i)
+                      x: new_x,
+                      y: new_y,
+                      buttons: @buttons)
       end
 
       self
@@ -140,16 +166,26 @@ module Ferrum
 
     def mouse_event(type:, button: :left, count: 1, modifiers: nil, wait: 0)
       button = validate_button(button)
-      options = { x: @x, y: @y, type: type, button: button, clickCount: count }
+      register_event_button(type, button)
+      options = { x: @x, y: @y, type: type, button: button, buttons: @buttons, clickCount: count }
       options.merge!(modifiers: modifiers) if modifiers
       @page.command("Input.dispatchMouseEvent", wait: wait, slowmoable: true, **options)
     end
 
     def validate_button(button)
       button = button.to_s
-      raise "Invalid button: #{button}" unless VALID_BUTTONS.include?(button)
+      raise "Invalid button: #{button}" unless BUTTON_MASKS.key?(button)
 
       button
     end
+
+    def register_event_button(type, button)
+      case type
+      when "mousePressed"
+        @buttons |= BUTTON_MASKS[button]
+      when "mouseReleased"
+        @buttons &= ~BUTTON_MASKS[button]
+      end
+    end
   end
 end

data/lib/ferrum/network/exchange.rb

@@ -28,6 +28,15 @@ module Ferrum
       # @return [Error, nil]
       attr_accessor :error
 
+      # Determines if the network exchange is unknown due to
+      # a lost of its context
+      #
+      # @return Boolean
+      attr_accessor :unknown
+
+      # @api private
+      attr_accessor :request_extra_info
+
       #
       # Initializes the network exchange.
       #
@@ -40,6 +49,8 @@ module Ferrum
         @page = page
         @intercepted_request = nil
         @request = @response = @error = nil
+        @request_extra_info = nil
+        @unknown = false
       end
 
       #
@@ -54,6 +65,15 @@ module Ferrum
         request&.type?(:document) && request&.frame_id == frame_id
       end
 
+      #
+      # The loader ID of the request.
+      #
+      # @return [String, nil]
+      #
+      def loader_id
+        request&.loader_id
+      end
+
       #
       # Determines if the network exchange has a request.
       #
@@ -74,12 +94,12 @@ module Ferrum
 
       #
       # Determines if the request was blocked, a response was returned, or if an
-      # error occurred.
+      # error occurred or the exchange is unknown and cannot be inferred.
       #
       # @return [Boolean]
       #
       def finished?
-        blocked? || response&.loaded? || !error.nil? || ping?
+        blocked? || response&.loaded? || !error.nil? || ping? || blob? || unknown
       end
 
       #
@@ -127,6 +147,15 @@ module Ferrum
         !!request&.ping?
       end
 
+      #
+      # Determines if the exchange is blob.
+      #
+      # @return [Boolean]
+      #
+      def blob?
+        !!url&.start_with?("blob:")
+      end
+
       #
       # Returns request's URL.
       #
@@ -156,7 +185,8 @@ module Ferrum
           "@intercepted_request=#{@intercepted_request.inspect} " \
           "@request=#{@request.inspect} " \
           "@response=#{@response.inspect} " \
-          "@error=#{@error.inspect}>"
+          "@error=#{@error.inspect}> " \
+          "@unknown=#{@unknown.inspect}>"
       end
     end
   end

data/lib/ferrum/network/request.rb

@@ -71,6 +71,15 @@ module Ferrum
         @params["frameId"]
       end
 
+      #
+      # The loader ID of the request.
+      #
+      # @return [String]
+      #
+      def loader_id
+        @params["loaderId"]
+      end
+
       #
       # The request timestamp.
       #

data/lib/ferrum/network.rb

@@ -100,7 +100,7 @@ module Ferrum
     #
     # Page request of the main frame.
     #
-    # @return [Request]
+    # @return [Request, nil]
     #
     # @example
     #   browser.go_to("https://github.com/")
@@ -380,15 +380,26 @@ module Ferrum
 
         # We can build exchange in two places, here on the event or when request
         # is interrupted. So we have to be careful when to create new one. We
-        # create new exchange only if there's no with such id or there's but
+        # create new exchange only if there's no with such id or there's, but
         # it's filled with request which means this one is new but has response
         # for a redirect. So we assign response from the params to previous
         # exchange and build new exchange to assign this request to it.
         exchange = select(request.id).last
-        exchange = build_exchange(request.id) unless exchange&.blank?
+        exchange = build_exchange(request.id) if exchange.nil? || !exchange.blank?
+        request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame.id)
+          @exchange = exchange
+          classify_pending_exchanges(exchange.loader_id)
+        end
+      end
+
+      @page.on("Network.requestWillBeSentExtraInfo") do |params|
+        exchange = select(params["requestId"]).last
+        exchange ||= build_exchange(params["requestId"])
+        exchange.request_extra_info = params
+        exchange.request&.headers&.merge!(params["headers"])
       end
     end
 
@@ -407,6 +418,8 @@ module Ferrum
         exchange = select(params["requestId"]).last
         next unless exchange
 
+        exchange.unknown = false
+
         if (response = exchange.response)
           response.loaded = true
           response.body_size = params["encodedDataLength"]
@@ -489,5 +502,17 @@ module Ferrum
     def whitelist?
       Array(@whitelist).any?
     end
+
+    # When the main frame navigates Chrome doesn't send `Network.loadingFailed`
+    # for pending async requests. Therefore, we mark pending connections as unknown since
+    # they are not relevant to the current navigation.
+    def classify_pending_exchanges(new_loader_id)
+      @traffic.each do |exchange|
+        break if exchange.loader_id == new_loader_id
+        next unless exchange.pending?
+
+        exchange.unknown = true
+      end
+    end
   end
 end

data/lib/ferrum/node.rb

@@ -76,6 +76,7 @@ module Ferrum
       when :double
         page.mouse.move(x: x, y: y)
         page.mouse.down(modifiers: modifiers, count: 2)
+        sleep(delay)
         page.mouse.up(modifiers: modifiers, count: 2)
       when :left
         page.mouse.click(x: x, y: y, modifiers: modifiers, delay: delay)

data/lib/ferrum/page/frames.rb

@@ -124,7 +124,7 @@ module Ferrum
         on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the
-          # tree and we save it in a variable because if we call that again root
+          # tree, and we save it in a variable because if we call that again root
           # node will change the id and all subsequent nodes have to change id too.
           if @main_frame.id == params["frameId"]
             @event.set if idling?
@@ -177,12 +177,16 @@ module Ferrum
           execution_id = params["executionContextId"]
           frame = frame_by(execution_id: execution_id)
           frame&.execution_id = nil
+          frame&.state = :stopped_loading
         end
       end
 
       def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
-          @frames.each_value { |f| f.execution_id = nil }
+          @frames.each_value do |f|
+            f.execution_id = nil
+            f.state = :stopped_loading
+          end
         end
       end
 

data/lib/ferrum/page/screencast.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    module Screencast
+      # Starts sending frames to record screencast to the given block.
+      #
+      # @param [Hash{Symbol => Object}] opts
+      #
+      # @option opts [:jpeg, :png] :format
+      #   The format the image should be returned in.
+      #
+      # @option opts [Integer] :quality
+      #   The image quality. **Note:** 0-100 works for JPEG only.
+      #
+      # @option opts [Integer] :max_width
+      #   Maximum screencast frame width.
+      #
+      # @option opts [Integer] :max_height
+      #   Maximum screencast frame height.
+      #
+      # @option opts [Integer] :every_nth_frame
+      #   Send every n-th frame.
+      #
+      # @yield [data, metadata, session_id]
+      #   The given block receives the screencast frame along with metadata
+      #   about the frame and the screencast session ID.
+      #
+      # @yieldparam data [String]
+      #   Base64-encoded compressed image.
+      #
+      # @yieldparam metadata [Hash{String => Object}]
+      #   Screencast frame metadata.
+      #
+      # @option metadata [Integer] 'offsetTop'
+      #   Top offset in DIP.
+      #
+      # @option metadata [Integer] 'pageScaleFactor'
+      #   Page scale factor.
+      #
+      # @option metadata [Integer] 'deviceWidth'
+      #   Device screen width in DIP.
+      #
+      # @option metadata [Integer] 'deviceHeight'
+      #   Device screen height in DIP.
+      #
+      # @option metadata [Integer] 'scrollOffsetX'
+      #   Position of horizontal scroll in CSS pixels.
+      #
+      # @option metadata [Integer] 'scrollOffsetY'
+      #   Position of vertical scroll in CSS pixels.
+      #
+      # @option metadata [Float] 'timestamp'
+      #   (optional) Frame swap timestamp in seconds since Unix epoch.
+      #
+      # @yieldparam session_id [Integer]
+      #   Frame number.
+      #
+      # @example
+      #   require "base64"
+      #
+      #   page.go_to("https://apple.com/ipad")
+      #
+      #   page.start_screencast(format: :jpeg, quality: 75) do |data, metadata|
+      #     timestamp = (metadata['timestamp'] * 1000).to_i
+      #     File.binwrite("image_#{timestamp}.jpg", Base64.decode64(data))
+      #   end
+      #
+      #   sleep 10
+      #
+      #   page.stop_screencast
+      #
+      def start_screencast(**opts)
+        options = opts.transform_keys { START_SCREENCAST_KEY_CONV.fetch(_1, _1) }
+        response = command("Page.startScreencast", **options)
+
+        if (error_text = response["errorText"]) # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
+          raise "Starting screencast failed (#{error_text})"
+        end
+
+        on("Page.screencastFrame") do |params|
+          data, metadata, session_id = params.values_at("data", "metadata", "sessionId")
+
+          command("Page.screencastFrameAck", sessionId: session_id)
+
+          yield data, metadata, session_id
+        end
+      end
+
+      # Stops sending frames.
+      def stop_screencast
+        command("Page.stopScreencast")
+      end
+
+      START_SCREENCAST_KEY_CONV = {
+        max_width: :maxWidth,
+        max_height: :maxHeight,
+        every_nth_frame: :everyNthFrame
+      }.freeze
+    end
+  end
+end

data/lib/ferrum/page.rb

@@ -10,6 +10,7 @@ require "ferrum/dialog"
 require "ferrum/network"
 require "ferrum/downloads"
 require "ferrum/page/frames"
+require "ferrum/page/screencast"
 require "ferrum/page/screenshot"
 require "ferrum/page/animation"
 require "ferrum/page/tracing"
@@ -27,6 +28,7 @@ module Ferrum
     delegate %i[base_url default_user_agent timeout timeout=] => :@options
 
     include Animation
+    include Screencast
     include Screenshot
     include Frames
     include Stream
@@ -393,6 +395,19 @@ module Ferrum
       end
     end
 
+    def off(name, id)
+      case name
+      when :dialog
+        client.off("Page.javascriptDialogOpening", id)
+      when :request
+        client.off("Fetch.requestPaused", id)
+      when :auth
+        client.off("Fetch.authRequired", id)
+      else
+        client.off(name, id)
+      end
+    end
+
     def subscribed?(event)
       client.subscribed?(event)
     end
@@ -475,7 +490,7 @@ module Ferrum
       # opens a new window for which `frameStoppedLoading` event never
       # occurs and thus search for nodes cannot be completed. Here we check
       # the history and if the transitionType for example `link` then
-      # content is already loaded and we can try to get the document.
+      # content is already loaded, and we can try to get the document.
       document_node_id
     end
 

data/lib/ferrum/target.rb

@@ -8,7 +8,8 @@ module Ferrum
     # where we enhance page class and build page ourselves.
     attr_writer :page
 
-    attr_reader :session_id, :options
+    attr_reader :options
+    attr_accessor :session_id
 
     def initialize(browser_client, session_id = nil, params = nil)
       @page = nil
@@ -67,11 +68,19 @@ module Ferrum
       !!opener_id
     end
 
+    def iframe?
+      type == "iframe"
+    end
+
     def maybe_sleep_if_new_window
       # Dirty hack because new window doesn't have events at all
       sleep(NEW_WINDOW_WAIT) if window?
     end
 
+    def command(...)
+      client.command(...)
+    end
+
     private
 
     def build_client

data/lib/ferrum/utils/elapsed_time.rb

@@ -9,6 +9,10 @@ module Ferrum
         @start ||= monotonic_time
       end
 
+      def reset
+        @start = monotonic_time
+      end
+
       def elapsed_time(start = nil)
         monotonic_time - (start || @start)
       end

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.16"
+  VERSION = "0.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: '0.16'
+  version: '0.17'
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-01 00:00:00.000000000 Z
+date: 2025-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -129,6 +129,7 @@ files:
 - lib/ferrum/page.rb
 - lib/ferrum/page/animation.rb
 - lib/ferrum/page/frames.rb
+- lib/ferrum/page/screencast.rb
 - lib/ferrum/page/screenshot.rb
 - lib/ferrum/page/stream.rb
 - lib/ferrum/page/tracing.rb