ferrum 0.11 → 0.12

data/lib/ferrum/page/screenshot.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "ferrum/rbga"
+require "ferrum/rgba"
 
 module Ferrum
   class Page
@@ -13,21 +13,19 @@ module Ferrum
       }.freeze
 
       PAPER_FORMATS = {
-        letter:   { width:  8.50, height: 11.00 },
-        legal:    { width:  8.50, height: 14.00 },
-        tabloid:  { width: 11.00, height: 17.00 },
-        ledger:   { width: 17.00, height: 11.00 },
-        A0:       { width: 33.10, height: 46.80 },
-        A1:       { width: 23.40, height: 33.10 },
-        A2:       { width: 16.54, height: 23.40 },
-        A3:       { width: 11.70, height: 16.54 },
-        A4:       { width:  8.27, height: 11.70 },
-        A5:       { width:  5.83, height:  8.27 },
-        A6:       { width:  4.13, height:  5.83 },
+        letter: { width: 8.50, height: 11.00 },
+        legal: { width: 8.50, height: 14.00 },
+        tabloid: { width: 11.00, height: 17.00 },
+        ledger: { width: 17.00, height: 11.00 },
+        A0: { width: 33.10, height: 46.80 },
+        A1: { width: 23.40, height: 33.10 },
+        A2: { width: 16.54, height: 23.40 },
+        A3: { width: 11.70, height: 16.54 },
+        A4: { width:  8.27, height: 11.70 },
+        A5: { width:  5.83, height:  8.27 },
+        A6: { width:  4.13, height:  5.83 }
       }.freeze
 
-      STREAM_CHUNK = 128 * 1024
-
       def screenshot(**opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
@@ -42,17 +40,13 @@ module Ferrum
         path, encoding = common_options(**opts)
         options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
         handle = command("Page.printToPDF", **options).fetch("stream")
-
-        if path
-          stream_to_file(handle, path: path)
-        else
-          stream_to_memory(handle, encoding: encoding)
-        end
+        stream_to(path: path, encoding: encoding, handle: handle)
       end
 
       def mhtml(path: nil)
         data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
         return data if path.nil?
+
         save_file(path, data)
       end
 
@@ -73,30 +67,8 @@ module Ferrum
 
       def save_file(path, data)
         return data unless path
-        File.open(path.to_s, "wb") { |f| f.write(data) }
-      end
 
-      def stream_to_file(handle, path:)
-        File.open(path, "wb") { |f| stream_to(handle, f) }
-        true
-      end
-
-      def stream_to_memory(handle, encoding:)
-        data = String.new("") # Mutable string has << and compatible to File
-        stream_to(handle, data)
-        encoding == :base64 ? Base64.encode64(data) : data
-      end
-
-      def stream_to(handle, output)
-        loop do
-          result = command("IO.read", handle: handle, size: STREAM_CHUNK)
-
-          data_chunk = result["data"]
-          data_chunk = Base64.decode64(data_chunk) if result["base64Encoded"]
-          output << data_chunk
-
-          break if result["eof"]
-        end
+        File.binwrite(path.to_s, data)
       end
 
       def common_options(encoding: :base64, path: nil, **_)
@@ -119,44 +91,57 @@ module Ferrum
                          paper_height: dimension[:height])
         end
 
-        options.map { |k, v| [to_camel_case(k), v] }.to_h
+        options.transform_keys { |k| to_camel_case(k) }
       end
 
-      def screenshot_options(path = nil, format: nil, scale: 1.0, **opts)
-        options = {}
+      def screenshot_options(path = nil, format: nil, scale: 1.0, **options)
+        screenshot_options = {}
+
+        format, quality = format_options(format, path, options[:quality])
+        screenshot_options.merge!(quality: quality) if quality
+        screenshot_options.merge!(format: format)
+
+        clip = area_options(options[:full], options[:selector], scale)
+        screenshot_options.merge!(clip: clip) if clip
+
+        screenshot_options
+      end
 
+      def format_options(format, path, quality)
         format ||= path ? File.extname(path).delete(".") : "png"
         format = "jpeg" if format == "jpg"
         raise "Not supported options `:format` #{format}. jpeg | png" if format !~ /jpeg|png/i
-        options.merge!(format: format)
 
-        options.merge!(quality: opts[:quality] ? opts[:quality] : 75) if format == "jpeg"
+        quality ||= 75 if format == "jpeg"
 
-        if !!opts[:full] && opts[:selector]
-          warn "Ignoring :selector in #screenshot since full: true was given at #{caller(1..1).first}"
-        end
+        [format, quality]
+      end
 
-        if !!opts[:full]
-          width, height = document_size
-          options.merge!(clip: { x: 0, y: 0, width: width, height: height, scale: scale }) if width > 0 && height > 0
-        elsif opts[:selector]
-          options.merge!(clip: get_bounding_rect(opts[:selector]).merge(scale: scale))
-        end
+      def area_options(full, selector, scale)
+        message = "Ignoring :selector in #screenshot since full: true was given at #{caller(1..1).first}"
+        warn(message) if full && selector
 
-        if scale != 1.0
-          if !options[:clip]
+        clip = if full
+                 width, height = document_size
+                 { x: 0, y: 0, width: width, height: height, scale: scale } if width.positive? && height.positive?
+               elsif selector
+                 bounding_rect(selector).merge(scale: scale)
+               end
+
+        if scale != 1
+          unless clip
             width, height = viewport_size
-            options[:clip] = { x: 0, y: 0, width: width, height: height }
+            clip = { x: 0, y: 0, width: width, height: height }
           end
 
-          options[:clip].merge!(scale: scale)
+          clip.merge!(scale: scale)
         end
 
-        options
+        clip
       end
 
-      def get_bounding_rect(selector)
-        rect = evaluate_async(%Q(
+      def bounding_rect(selector)
+        rect = evaluate_async(%(
           const rect = document
                          .querySelector('#{selector}')
                          .getBoundingClientRect();
@@ -169,7 +154,8 @@ module Ferrum
 
       def to_camel_case(option)
         return :preferCSSPageSize if option == :prefer_css_page_size
-        option.to_s.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.to_sym
+
+        option.to_s.gsub(%r{(?:_|(/))([a-z\d]*)}) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }.to_sym
       end
 
       def capture_screenshot(options, full, background_color)

data/lib/ferrum/page/stream.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    module Stream
+      STREAM_CHUNK = 128 * 1024
+
+      def stream_to(path:, encoding:, handle:)
+        if path.nil?
+          stream_to_memory(encoding: encoding, handle: handle)
+        else
+          stream_to_file(path: path, handle: handle)
+        end
+      end
+
+      def stream_to_file(path:, handle:)
+        File.open(path, "wb") { |f| stream(output: f, handle: handle) }
+        true
+      end
+
+      def stream_to_memory(encoding:, handle:)
+        data = String.new # Mutable string has << and compatible to File
+        stream(output: data, handle: handle)
+        encoding == :base64 ? Base64.encode64(data) : data
+      end
+
+      def stream(output:, handle:)
+        loop do
+          result = command("IO.read", handle: handle, size: STREAM_CHUNK)
+          chunk = result.fetch("data")
+          chunk = Base64.decode64(chunk) if result["base64Encoded"]
+          output << chunk
+          break if result["eof"]
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/page/tracing.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Page
+    class Tracing
+      EXCLUDED_CATEGORIES = %w[*].freeze
+      SCREENSHOT_CATEGORIES = %w[disabled-by-default-devtools.screenshot].freeze
+      INCLUDED_CATEGORIES = %w[devtools.timeline v8.execute disabled-by-default-devtools.timeline
+                               disabled-by-default-devtools.timeline.frame toplevel blink.console
+                               blink.user_timing latencyInfo disabled-by-default-devtools.timeline.stack
+                               disabled-by-default-v8.cpu_profiler disabled-by-default-v8.cpu_profiler.hires].freeze
+      DEFAULT_TRACE_CONFIG = {
+        includedCategories: INCLUDED_CATEGORIES,
+        excludedCategories: EXCLUDED_CATEGORIES
+      }.freeze
+
+      def initialize(page)
+        @page = page
+        @subscribed_tracing_complete = false
+      end
+
+      def record(path: nil, encoding: :binary, timeout: nil, trace_config: nil, screenshots: false)
+        @path = path
+        @encoding = encoding
+        @pending = Concurrent::IVar.new
+        trace_config ||= DEFAULT_TRACE_CONFIG.dup
+
+        if screenshots
+          included = trace_config.fetch(:includedCategories, [])
+          trace_config.merge!(includedCategories: included | SCREENSHOT_CATEGORIES)
+        end
+
+        subscribe_tracing_complete
+
+        start(trace_config)
+        yield
+        stop
+
+        @pending.value!(timeout || @page.timeout)
+      end
+
+      private
+
+      def start(config)
+        @page.command("Tracing.start", transferMode: "ReturnAsStream", traceConfig: config)
+      end
+
+      def stop
+        @page.command("Tracing.end")
+      end
+
+      def subscribe_tracing_complete
+        return if @subscribed_tracing_complete
+
+        @page.on("Tracing.tracingComplete") do |event, index|
+          next if index.to_i != 0
+
+          @pending.set(stream_handle(event["stream"]))
+        rescue StandardError => e
+          @pending.fail(e)
+        end
+
+        @subscribed_tracing_complete = true
+      end
+
+      def stream_handle(handle)
+        @page.stream_to(path: @path, encoding: @encoding, handle: handle)
+      end
+    end
+  end
+end

data/lib/ferrum/page.rb

@@ -10,6 +10,8 @@ require "ferrum/network"
 require "ferrum/page/frames"
 require "ferrum/page/screenshot"
 require "ferrum/page/animation"
+require "ferrum/page/tracing"
+require "ferrum/page/stream"
 require "ferrum/browser/client"
 
 module Ferrum
@@ -32,21 +34,26 @@ module Ferrum
 
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
-                current_url current_title url title body doctype set_content
+                current_url current_title url title body doctype content=
                 execution_id evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
 
-    include Frames, Screenshot, Animation
+    include Animation
+    include Screenshot
+    include Frames
+    include Stream
 
     attr_accessor :referrer
     attr_reader :target_id, :browser,
                 :headers, :cookies, :network,
-                :mouse, :keyboard, :event, :document_id
+                :mouse, :keyboard, :event,
+                :tracing
 
     def initialize(target_id, browser)
       @frames = {}
       @main_frame = Frame.new(nil, self)
-      @target_id, @browser = target_id, browser
+      @browser = browser
+      @target_id = target_id
       @event = Event.new.tap(&:set)
 
       host = @browser.process.host
@@ -54,9 +61,12 @@ module Ferrum
       ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
       @client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
 
-      @mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
-      @headers, @cookies = Headers.new(self), Cookies.new(self)
+      @mouse = Mouse.new(self)
+      @keyboard = Keyboard.new(self)
+      @headers = Headers.new(self)
+      @cookies = Cookies.new(self)
       @network = Network.new(self)
+      @tracing = Tracing.new(self)
 
       subscribe
       prepare_page
@@ -66,6 +76,10 @@ module Ferrum
       @browser.timeout
     end
 
+    def context
+      @browser.contexts.find_by(target_id: target_id)
+    end
+
     def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
@@ -77,6 +91,7 @@ module Ferrum
             net::ERR_CONNECTION_TIMED_OUT].include?(response["errorText"])
         raise StatusError, options[:url]
       end
+
       response["frameId"]
     rescue TimeoutError
       if @browser.pending_connection_errors
@@ -85,6 +100,7 @@ module Ferrum
       end
     end
     alias goto go_to
+    alias go go_to
 
     def close
       @headers.clear
@@ -113,14 +129,16 @@ module Ferrum
       @browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
     end
 
-    def position=(left:, top:)
-      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: { left: left, top: top })
+    def position=(options)
+      @browser.command("Browser.setWindowBounds",
+                       windowId: window_id,
+                       bounds: { left: options[:left], top: options[:top] })
     end
 
     def refresh
       command("Page.reload", wait: timeout, slowmoable: true)
     end
-    alias_method :reload, :refresh
+    alias reload refresh
 
     def stop
       command("Page.stopLoading", slowmoable: true)
@@ -140,8 +158,7 @@ module Ferrum
       @event.set
     end
 
-    def bypass_csp(value = true)
-      enabled = !!value
+    def bypass_csp(enabled: true)
       command("Page.setBypassCSP", enabled: enabled)
       enabled
     end
@@ -155,14 +172,15 @@ module Ferrum
     end
 
     def command(method, wait: 0, slowmoable: false, **params)
-      iteration = @event.reset if wait > 0
-      sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
+      iteration = @event.reset if wait.positive?
+      sleep(@browser.slowmo) if slowmoable && @browser.slowmo.positive?
       result = @client.command(method, params)
 
-      if wait > 0
-        @event.wait(wait) # Wait a bit after command and check if iteration has
-                          # changed which means there was some network event for
-                          # the main frame and it started to load new content.
+      if wait.positive?
+        @event.wait(wait)
+        # Wait a bit after command and check if iteration has
+        # changed which means there was some network event for
+        # the main frame and it started to load new content.
         if iteration != @event.iteration
           set = @event.wait(@browser.timeout)
           raise TimeoutError unless set
@@ -214,8 +232,19 @@ module Ferrum
 
       if @browser.js_errors
         on("Runtime.exceptionThrown") do |params|
-          # FIXME https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
-          Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
+          # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+          Thread.main.raise JavaScriptError.new(
+            params.dig("exceptionDetails", "exception"),
+            params.dig("exceptionDetails", "stackTrace")
+          )
+        end
+      end
+
+      on(:dialog) do |dialog, _index, total|
+        if total == 1
+          warn "Dialog was shown but you didn't provide `on(:dialog)` callback, accepting it by default. " \
+               "Please take a look at https://github.com/rubycdp/ferrum#dialog"
+          dialog.accept
         end
       end
     end
@@ -228,8 +257,22 @@ module Ferrum
       command("Log.enable")
       command("Network.enable")
 
+      if @browser.proxy_options && @browser.proxy_options[:user] && @browser.proxy_options[:password]
+        auth_options = @browser.proxy_options.slice(:user, :password)
+        network.authorize(type: :proxy, **auth_options) do |request, _index, _total|
+          request.continue
+        end
+      end
+
       if @browser.options[:save_path]
-        command("Page.setDownloadBehavior", behavior: "allow", downloadPath: @browser.options[:save_path])
+        unless Pathname.new(@browser.options[:save_path]).absolute?
+          raise Error, "supply absolute path for `:save_path` option"
+        end
+
+        @browser.command("Browser.setDownloadBehavior",
+                         browserContextId: context.id,
+                         downloadPath: browser.options[:save_path],
+                         behavior: "allow", eventsEnabled: true)
       end
 
       @browser.extensions.each do |extension|
@@ -242,14 +285,14 @@ module Ferrum
       resize(width: width, height: height)
 
       response = command("Page.getNavigationHistory")
-      if response.dig("entries", 0, "transitionType") != "typed"
-        # If we create page by clicking links, submiting forms and so on it
-        # opens a new window for which `frameStoppedLoading` event never
-        # occurs and thus search for nodes cannot be completed. Here we check
-        # the history and if the transitionType for example `link` then
-        # content is already loaded and we can try to get the document.
-        get_document_id
-      end
+      return unless response.dig("entries", 0, "transitionType") != "typed"
+
+      # If we create page by clicking links, submitting forms and so on it
+      # opens a new window for which `frameStoppedLoading` event never
+      # occurs and thus search for nodes cannot be completed. Here we check
+      # the history and if the transitionType for example `link` then
+      # content is already loaded and we can try to get the document.
+      document_node_id
     end
 
     def inject_extensions
@@ -268,13 +311,15 @@ module Ferrum
     def history_navigate(delta:)
       history = command("Page.getNavigationHistory")
       index, entries = history.values_at("currentIndex", "entries")
+      entry = entries[index + delta]
 
-      if entry = entries[index + delta]
-        # Potential wait because of network event
-        command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
-                                               slowmoable: true,
-                                               entryId: entry["id"])
-      end
+      return unless entry
+
+      # Potential wait because of network event
+      command("Page.navigateToHistoryEntry",
+              wait: Mouse::CLICK_WAIT,
+              slowmoable: true,
+              entryId: entry["id"])
     end
 
     def combine_url!(url_or_path)
@@ -288,8 +333,8 @@ module Ferrum
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
     end
 
-    def get_document_id
-      @document_id = command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    def document_node_id
+      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
     end
   end
 end

data/lib/ferrum/proxy.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "webrick"
+require "webrick/httpproxy"
+
+module Ferrum
+  class Proxy
+    def self.start(**args)
+      new(**args).tap(&:start)
+    end
+
+    attr_reader :host, :port, :user, :password
+
+    def initialize(host: "127.0.0.1", port: 0, user: nil, password: nil)
+      @file = nil
+      @host = host
+      @port = port
+      @user = user
+      @password = password
+      at_exit { stop }
+    end
+
+    def start
+      options = {
+        ProxyURI: nil, ServerType: Thread,
+        Logger: Logger.new(IO::NULL), AccessLog: [],
+        BindAddress: host, Port: port
+      }
+
+      if user && password
+        @file = Tempfile.new("htpasswd")
+        htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@file.path)
+        htpasswd.set_passwd "Proxy Realm", user, password
+        htpasswd.flush
+        authenticator = WEBrick::HTTPAuth::ProxyBasicAuth.new(Realm: "Proxy Realm",
+                                                              UserDB: htpasswd,
+                                                              Logger: Logger.new(IO::NULL))
+        options.merge!(ProxyAuthProc: authenticator.method(:authenticate).to_proc)
+      end
+
+      @server = WEBrick::HTTPProxyServer.new(**options)
+      @server.start
+      @port = @server.config[:Port]
+    end
+
+    def rotate(host:, port:, user: nil, password: nil)
+      credentials = "#{user}:#{password}@" if user && password
+      proxy_uri = "schema://#{credentials}#{host}:#{port}"
+      @server.config[:ProxyURI] = URI.parse(proxy_uri)
+    end
+
+    def stop
+      @file&.unlink
+      @server.shutdown
+    end
+  end
+end

data/lib/ferrum/{rbga.rb → rgba.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Ferrum
   class RGBA
     def initialize(red, green, blue, alpha)
@@ -23,13 +25,13 @@ module Ferrum
     end
 
     def validate_color(value)
-      return if value && value.is_a?(Integer) && Range.new(0, 255).include?(value)
+      return if value.is_a?(Integer) && Range.new(0, 255).include?(value)
 
       raise ArgumentError, "Wrong value of #{value} should be Integer from 0 to 255"
     end
 
     def validate_alpha
-      return if alpha && alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
+      return if alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
 
       raise ArgumentError,
             "Wrong alpha value #{alpha} should be Float between 0.0 (fully transparent) and 1.0 (fully opaque)"

data/lib/ferrum/target.rb

@@ -9,6 +9,7 @@ module Ferrum
     attr_writer :page
 
     def initialize(browser, params = nil)
+      @page = nil
       @browser = browser
       @params = params
     end

data/lib/ferrum/utils/attempt.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ferrum
+  module Utils
+    module Attempt
+      module_function
+
+      def with_retry(errors:, max:, wait:)
+        attempts ||= 1
+        yield
+      rescue *Array(errors)
+        raise if attempts >= max
+
+        attempts += 1
+        sleep(wait)
+        retry
+      end
+    end
+  end
+end

data/lib/ferrum/utils/elapsed_time.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+
+module Ferrum
+  module Utils
+    module ElapsedTime
+      module_function
+
+      def start
+        @start ||= monotonic_time
+      end
+
+      def elapsed_time(start = nil)
+        monotonic_time - (start || @start)
+      end
+
+      def monotonic_time
+        Concurrent.monotonic_time
+      end
+
+      def timeout?(start, timeout)
+        elapsed_time(start) > timeout
+      end
+    end
+  end
+end