ferrum 0.6.2 → 0.10.1

data/lib/ferrum/page.rb

@@ -13,6 +13,8 @@ require "ferrum/browser/client"
 
 module Ferrum
   class Page
+    GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
+
     class Event < Concurrent::Event
       def iteration
         synchronize { @iteration }
@@ -29,8 +31,8 @@ module Ferrum
 
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
-                current_url current_title url title body doctype
-                execution_id evaluate evaluate_on evaluate_async execute
+                current_url current_title url title body doctype set_content
+                execution_id evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
 
     include Frames, Screenshot
@@ -42,13 +44,14 @@ module Ferrum
 
     def initialize(target_id, browser)
       @frames = {}
+      @main_frame = Frame.new(nil, self)
       @target_id, @browser = target_id, browser
       @event = Event.new.tap(&:set)
 
       host = @browser.process.host
       port = @browser.process.port
       ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
-      @client = Browser::Client.new(browser, ws_url, 1000)
+      @client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
 
       @mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
       @headers, @cookies = Headers.new(self), Cookies.new(self)
@@ -62,10 +65,10 @@ module Ferrum
       @browser.timeout
     end
 
-    def goto(url = nil)
+    def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
-      response = command("Page.navigate", wait: timeout, **options)
+      response = command("Page.navigate", wait: GOTO_WAIT, **options)
       # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
       if %w[net::ERR_NAME_NOT_RESOLVED
             net::ERR_NAME_RESOLUTION_FAILED
@@ -74,7 +77,13 @@ module Ferrum
         raise StatusError, options[:url]
       end
       response["frameId"]
+    rescue TimeoutError
+      if @browser.pending_connection_errors
+        pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
+        raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
+      end
     end
+    alias goto go_to
 
     def close
       @headers.clear
@@ -83,18 +92,16 @@ module Ferrum
     end
 
     def resize(width: nil, height: nil, fullscreen: false)
-      result = @browser.command("Browser.getWindowForTarget", targetId: @target_id)
-      @window_id, @bounds = result.values_at("windowId", "bounds")
-
       if fullscreen
         width, height = document_size
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "fullscreen" })
+        set_window_bounds(windowState: "fullscreen")
       else
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "normal" })
-        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { width: width, height: height, windowState: "normal" })
+        set_window_bounds(windowState: "normal")
+        set_window_bounds(width: width, height: height)
       end
 
-      command("Emulation.setDeviceMetricsOverride", width: width,
+      command("Emulation.setDeviceMetricsOverride", slowmoable: true,
+                                                    width: width,
                                                     height: height,
                                                     deviceScaleFactor: 1,
                                                     mobile: false,
@@ -102,7 +109,12 @@ module Ferrum
     end
 
     def refresh
-      command("Page.reload", wait: timeout)
+      command("Page.reload", wait: timeout, slowmoable: true)
+    end
+    alias_method :reload, :refresh
+
+    def stop
+      command("Page.stopLoading", slowmoable: true)
     end
 
     def back
@@ -113,12 +125,39 @@ module Ferrum
       history_navigate(delta: 1)
     end
 
-    def command(method, wait: 0, **params)
+    def wait_for_reload(sec = 1)
+      @event.reset if @event.set?
+      @event.wait(sec)
+      @event.set
+    end
+
+    def bypass_csp(value = true)
+      enabled = !!value
+      command("Page.setBypassCSP", enabled: enabled)
+      enabled
+    end
+
+    def window_id
+      @browser.command("Browser.getWindowForTarget", targetId: @target_id)["windowId"]
+    end
+
+    def set_window_bounds(bounds = {})
+      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: bounds)
+    end
+
+    def command(method, wait: 0, slowmoable: false, **params)
       iteration = @event.reset if wait > 0
+      sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
       result = @client.command(method, params)
+
       if wait > 0
-        @event.wait(wait)
-        @event.wait(@browser.timeout) if iteration != @event.iteration
+        @event.wait(wait) # Wait a bit after command and check if iteration has
+                          # changed which means there was some network event for
+                          # the main frame and it started to load new content.
+        if iteration != @event.iteration
+          set = @event.wait(@browser.timeout)
+          raise TimeoutError unless set
+        end
       end
       result
     end
@@ -133,6 +172,9 @@ module Ferrum
       when :request
         @client.on("Fetch.requestPaused") do |params, index, total|
           request = Network::InterceptedRequest.new(self, params)
+          exchange = network.select(request.network_id).last
+          exchange ||= network.build_exchange(request.network_id)
+          exchange.intercepted_request = request
           block.call(request, index, total)
         end
       when :auth
@@ -145,6 +187,10 @@ module Ferrum
       end
     end
 
+    def subscribed?(event)
+      @client.subscribed?(event)
+    end
+
     private
 
     def subscribe
@@ -163,14 +209,6 @@ module Ferrum
           Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
         end
       end
-
-      on("Page.domContentEventFired") do |params|
-        # `frameStoppedLoading` doesn't occur if status isn't success
-        if network.status != 200
-          @event.set
-          get_document_id
-        end
-      end
     end
 
     def prepare_page
@@ -224,7 +262,9 @@ module Ferrum
 
       if entry = entries[index + delta]
         # Potential wait because of network event
-        command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT, entryId: entry["id"])
+        command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
+                                               slowmoable: true,
+                                               entryId: entry["id"])
       end
     end
 
@@ -233,7 +273,7 @@ module Ferrum
       nil_or_relative = url.nil? || url.relative?
 
       if nil_or_relative && !@browser.base_url
-        raise "Set :base_url browser's option or use absolute url in `goto`, you passed: #{url_or_path}"
+        raise "Set :base_url browser's option or use absolute url in `go_to`, you passed: #{url_or_path}"
       end
 
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s

data/lib/ferrum/page/frames.rb

@@ -40,18 +40,6 @@ module Ferrum
           frame.name = name unless name.to_s.empty?
         end
 
-        on("Page.frameScheduledNavigation") do |params|
-          frame = @frames[params["frameId"]]
-          frame.state = :scheduled_navigation
-          @event.reset
-        end
-
-        on("Page.frameClearedScheduledNavigation") do |params|
-          frame = @frames[params["frameId"]]
-          frame.state = :cleared_scheduled_navigation
-          @event.set if idling?
-        end
-
         on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the
@@ -85,21 +73,30 @@ module Ferrum
         on("Runtime.executionContextCreated") do |params|
           context_id = params.dig("context", "id")
           frame_id = params.dig("context", "auxData", "frameId")
+
+          unless @main_frame.id
+            root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
+            if frame_id == root_frame
+              @main_frame.id = frame_id
+              @frames[frame_id] = @main_frame
+            end
+          end
+
           frame = @frames[frame_id] || Frame.new(frame_id, self)
-          frame.execution_id = context_id
+          frame.set_execution_id(context_id)
 
-          @main_frame ||= frame
           @frames[frame_id] ||= frame
         end
 
         on("Runtime.executionContextDestroyed") do |params|
           execution_id = params["executionContextId"]
           frame = frames.find { |f| f.execution_id?(execution_id) }
-          frame.execution_id = nil
+          frame.reset_execution_id
         end
 
         on("Runtime.executionContextsCleared") do
           @frames.delete_if { |_, f| !f.main? }
+          @main_frame.reset_execution_id
         end
       end
 

data/lib/ferrum/page/screenshot.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "ferrum/rbga"
+
 module Ferrum
   class Page
     module Screenshot
@@ -24,19 +26,33 @@ module Ferrum
         A6:       { width:  4.13, height:  5.83 },
       }.freeze
 
+      STREAM_CHUNK = 128 * 1024
+
       def screenshot(**opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
-        data = capture_screenshot(options, opts[:full])
+        data = capture_screenshot(options, opts[:full], opts[:background_color])
         return data if encoding == :base64
-        save_file(path, data)
+
+        bin = Base64.decode64(data)
+        save_file(path, bin)
       end
 
       def pdf(**opts)
         path, encoding = common_options(**opts)
-        options = pdf_options(**opts)
-        data = command("Page.printToPDF", **options).fetch("data")
-        return data if encoding == :base64
+        options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
+        handle = command("Page.printToPDF", **options).fetch("stream")
+
+        if path
+          stream_to_file(handle, path: path)
+        else
+          stream_to_memory(handle, encoding: encoding)
+        end
+      end
+
+      def mhtml(path: nil)
+        data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
+        return data if path.nil?
         save_file(path, data)
       end
 
@@ -48,17 +64,39 @@ module Ferrum
 
       def document_size
         evaluate <<~JS
-          [document.documentElement.offsetWidth,
-           document.documentElement.offsetHeight]
+          [document.documentElement.scrollWidth,
+           document.documentElement.scrollHeight]
         JS
       end
 
       private
 
       def save_file(path, data)
-        bin = Base64.decode64(data)
-        return bin unless path
-        File.open(path.to_s, "wb") { |f| f.write(bin) }
+        return data unless path
+        File.open(path.to_s, "wb") { |f| f.write(data) }
+      end
+
+      def stream_to_file(handle, path:)
+        File.open(path, "wb") { |f| stream_to(handle, f) }
+        true
+      end
+
+      def stream_to_memory(handle, encoding:)
+        data = String.new("") # Mutable string has << and compatible to File
+        stream_to(handle, data)
+        encoding == :base64 ? Base64.encode64(data) : data
+      end
+
+      def stream_to(handle, output)
+        loop do
+          result = command("IO.read", handle: handle, size: STREAM_CHUNK)
+
+          data_chunk = result["data"]
+          data_chunk = Base64.decode64(data_chunk) if result["base64Encoded"]
+          output << data_chunk
+
+          break if result["eof"]
+        end
       end
 
       def common_options(encoding: :base64, path: nil, **_)
@@ -134,9 +172,11 @@ module Ferrum
         option.to_s.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.to_sym
       end
 
-      def capture_screenshot(options, full)
+      def capture_screenshot(options, full, background_color)
         maybe_resize_fullscreen(full) do
-          command("Page.captureScreenshot", options)
+          with_background_color(background_color) do
+            command("Page.captureScreenshot", **options)
+          end
         end.fetch("data")
       end
 
@@ -150,6 +190,18 @@ module Ferrum
       ensure
         resize(width: width, height: height) if full
       end
+
+      def with_background_color(color)
+        if color
+          raise ArgumentError, "Accept Ferrum::RGBA class only" unless color.is_a?(RGBA)
+
+          command("Emulation.setDefaultBackgroundColorOverride", color: color.to_h)
+        end
+
+        yield
+      ensure
+        command("Emulation.setDefaultBackgroundColorOverride") if color
+      end
     end
   end
 end

data/lib/ferrum/rbga.rb

@@ -0,0 +1,38 @@
+module Ferrum
+  class RGBA
+    def initialize(red, green, blue, alpha)
+      self.red = red
+      self.green = green
+      self.blue = blue
+      self.alpha = alpha
+
+      validate
+    end
+
+    def to_h
+      { r: red, g: green, b: blue, a: alpha }
+    end
+
+    private
+
+    attr_accessor :red, :green, :blue, :alpha
+
+    def validate
+      [red, green, blue].each(&method(:validate_color))
+      validate_alpha
+    end
+
+    def validate_color(value)
+      return if value && value.is_a?(Integer) && Range.new(0, 255).include?(value)
+
+      raise ArgumentError, "Wrong value of #{value} should be Integer from 0 to 255"
+    end
+
+    def validate_alpha
+      return if alpha && alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
+
+      raise ArgumentError,
+            "Wrong alpha value #{alpha} should be Float between 0.0 (fully transparent) and 1.0 (fully opaque)"
+    end
+  end
+end

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.6.2"
+  VERSION = "0.10.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.10.1
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-30 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: websocket-driver
@@ -64,28 +64,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -182,9 +182,14 @@ files:
 - lib/ferrum.rb
 - lib/ferrum/browser.rb
 - lib/ferrum/browser/client.rb
+- lib/ferrum/browser/command.rb
+- lib/ferrum/browser/options/base.rb
+- lib/ferrum/browser/options/chrome.rb
+- lib/ferrum/browser/options/firefox.rb
 - lib/ferrum/browser/process.rb
 - lib/ferrum/browser/subscriber.rb
 - lib/ferrum/browser/web_socket.rb
+- lib/ferrum/browser/xvfb.rb
 - lib/ferrum/context.rb
 - lib/ferrum/contexts.rb
 - lib/ferrum/cookies.rb
@@ -207,6 +212,7 @@ files:
 - lib/ferrum/page.rb
 - lib/ferrum/page/frames.rb
 - lib/ferrum/page/screenshot.rb
+- lib/ferrum/rbga.rb
 - lib/ferrum/target.rb
 - lib/ferrum/version.rb
 homepage: https://github.com/route/ferrum
@@ -228,7 +234,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Ruby headless Chrome driver