ferrum 0.12 → 0.14

data/lib/ferrum/page/frames.rb

@@ -5,12 +5,54 @@ require "ferrum/frame"
 module Ferrum
   class Page
     module Frames
+      # The page's main frame, the top of the tree and the parent of all frames.
+      #
+      # @return [Frame]
       attr_reader :main_frame
 
+      #
+      # Returns all the frames current page have.
+      #
+      # @return [Array<Frame>]
+      #
+      # @example
+      #   browser.go_to("https://www.w3schools.com/tags/tag_frame.asp")
+      #   browser.frames # =>
+      #   # [
+      #   #   #<Ferrum::Frame
+      #   #     @id="C6D104CE454A025FBCF22B98DE612B12"
+      #   #     @parent_id=nil @name=nil @state=:stopped_loading @execution_id=1>,
+      #   #   #<Ferrum::Frame
+      #   #     @id="C09C4E4404314AAEAE85928EAC109A93"
+      #   #     @parent_id="C6D104CE454A025FBCF22B98DE612B12" @state=:stopped_loading @execution_id=2>,
+      #   #   #<Ferrum::Frame
+      #   #     @id="2E9C7F476ED09D87A42F2FEE3C6FBC3C"
+      #   #     @parent_id="C6D104CE454A025FBCF22B98DE612B12" @state=:stopped_loading @execution_id=3>,
+      #   #   ...
+      #   # ]
+      #
       def frames
         @frames.values
       end
 
+      #
+      # Find frame by given options.
+      #
+      # @param [String] id
+      #   Unique frame's id that browser provides.
+      #
+      # @param [String] name
+      #   Frame's name if there's one.
+      #
+      # @param [String] execution_id
+      #   Frame's context execution id.
+      #
+      # @return [Frame, nil]
+      #   The matching frame.
+      #
+      # @example
+      #   browser.frame_by(id: "C6D104CE454A025FBCF22B98DE612B12")
+      #
       def frame_by(id: nil, name: nil, execution_id: nil)
         if id
           @frames[id]
@@ -25,6 +67,7 @@ module Ferrum
 
       def frames_subscribe
         subscribe_frame_attached
+        subscribe_frame_detached
         subscribe_frame_started_loading
         subscribe_frame_navigated
         subscribe_frame_stopped_loading
@@ -43,14 +86,26 @@ module Ferrum
       def subscribe_frame_attached
         on("Page.frameAttached") do |params|
           parent_frame_id, frame_id = params.values_at("parentFrameId", "frameId")
-          @frames[frame_id] = Frame.new(frame_id, self, parent_frame_id)
+          @frames.put_if_absent(frame_id, Frame.new(frame_id, self, parent_frame_id))
+        end
+      end
+
+      def subscribe_frame_detached
+        on("Page.frameDetached") do |params|
+          frame = @frames[params["frameId"]]
+
+          if frame&.main?
+            frame.execution_id = nil
+          else
+            @frames.delete(params["frameId"])
+          end
         end
       end
 
       def subscribe_frame_started_loading
         on("Page.frameStartedLoading") do |params|
           frame = @frames[params["frameId"]]
-          frame.state = :started_loading
+          frame.state = :started_loading if frame
           @event.reset
         end
       end
@@ -59,8 +114,11 @@ module Ferrum
         on("Page.frameNavigated") do |params|
           frame_id, name = params["frame"]&.values_at("id", "name")
           frame = @frames[frame_id]
-          frame.state = :navigated
-          frame.name = name unless name.to_s.empty?
+
+          if frame
+            frame.state = :navigated
+            frame.name = name
+          end
         end
       end
 
@@ -107,14 +165,12 @@ module Ferrum
             root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
             if frame_id == root_frame
               @main_frame.id = frame_id
-              @frames[frame_id] = @main_frame
+              @frames.put_if_absent(frame_id, @main_frame)
             end
           end
 
-          frame = @frames[frame_id] || Frame.new(frame_id, self)
+          frame = @frames.fetch_or_store(frame_id, Frame.new(frame_id, self))
           frame.execution_id = context_id
-
-          @frames[frame_id] ||= frame
         end
       end
 
@@ -128,13 +184,12 @@ module Ferrum
 
       def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
-          @frames.delete_if { |_, f| !f.main? }
-          @main_frame.execution_id = nil
+          @frames.each_value { |f| f.execution_id = nil }
         end
       end
 
       def idling?
-        @frames.all? { |_, f| f.state == :stopped_loading }
+        @frames.values.all? { |f| f.state == :stopped_loading }
       end
     end
   end

data/lib/ferrum/page/screenshot.rb

@@ -26,6 +26,51 @@ module Ferrum
         A6: { width:  4.13, height:  5.83 }
       }.freeze
 
+      #
+      # Saves screenshot on a disk or returns it as base64.
+      #
+      # @param [Hash{Symbol => Object}] opts
+      #
+      # @option opts [String] :path
+      #  The path to save a screenshot on the disk. `:encoding` will be set to
+      #  `:binary` automatically.
+      #
+      # @option opts [:base64, :binary] :encoding
+      #   The encoding the image should be returned in.
+      #
+      # @option opts ["jpeg", "png"] :format
+      #   The format the image should be returned in.
+      #
+      # @option opts [Integer] :quality
+      #   The image quality. **Note:** 0-100 works for jpeg only.
+      #
+      # @option opts [Boolean] :full
+      #   Whether you need full page screenshot or a viewport.
+      #
+      # @option opts [String] :selector
+      #   CSS selector for the given element.
+      #
+      # @option opts [Float] :scale
+      #   Zoom in/out.
+      #
+      # @option opts [Ferrum::RGBA] :background_color
+      #   Sets the background color.
+      #
+      # @example
+      #   browser.go_to("https://google.com/")
+      #
+      # @example Save on the disk in PNG:
+      #   browser.screenshot(path: "google.png") # => 134660
+      #
+      # @example Save on the disk in JPG:
+      #   browser.screenshot(path: "google.jpg") # => 30902
+      #
+      # @example Save to Base64 the whole page not only viewport and reduce quality:
+      #   browser.screenshot(full: true, quality: 60) # "iVBORw0KGgoAAAANS...
+      #
+      # @example Save with specific background color:
+      #   browser.screenshot(background_color: Ferrum::RGBA.new(0, 0, 0, 0.0))
+      #
       def screenshot(**opts)
         path, encoding = common_options(**opts)
         options = screenshot_options(path, **opts)
@@ -36,6 +81,42 @@ module Ferrum
         save_file(path, bin)
       end
 
+      #
+      # Saves PDF on a disk or returns it as Base64.
+      #
+      # @param [Hash{Symbol => Object}] opts
+      #
+      # @option opts [String] :path
+      #  The path to save a screenshot on the disk. `:encoding` will be set to
+      #  `:binary` automatically.
+      #
+      # @option opts [:base64, :binary] :encoding
+      #   The encoding the image should be returned in.
+      #
+      # @option opts [Boolean] :landscape (false)
+      #   Page orientation.
+      #
+      # @option opts [Float] :scale
+      #   Zoom in/out.
+      #
+      # @option opts [:letter, :legal, :tabloid, :ledger, :A0, :A1, :A2, :A3, :A4, :A5, :A6] :format
+      #   The standard paper size.
+      #
+      # @option opts [Float] :paper_width
+      #   Sets the paper's width.
+      #
+      # @option opts [Float] :paper_height
+      #   Sets the paper's height.
+      #
+      # @note
+      #   See other [native options](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-printToPDF) you
+      #   can pass.
+      #
+      # @example
+      #   browser.go_to("https://google.com/")
+      #   # Save to disk as a PDF
+      #   browser.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => true
+      #
       def pdf(**opts)
         path, encoding = common_options(**opts)
         options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
@@ -43,6 +124,16 @@ module Ferrum
         stream_to(path: path, encoding: encoding, handle: handle)
       end
 
+      #
+      # Saves MHTML on a disk or returns it as a string.
+      #
+      # @param [String, nil] path
+      #   The path to save a file on the disk.
+      #
+      # @example
+      #   browser.go_to("https://google.com/")
+      #   browser.mhtml(path: "google.mhtml") # => 87742
+      #
       def mhtml(path: nil)
         data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
         return data if path.nil?
@@ -56,6 +147,12 @@ module Ferrum
         JS
       end
 
+      def device_pixel_ratio
+        evaluate <<~JS
+          window.devicePixelRatio
+        JS
+      end
+
       def document_size
         evaluate <<~JS
           [document.documentElement.scrollWidth,

data/lib/ferrum/page/tracing.rb

@@ -19,6 +19,32 @@ module Ferrum
         @subscribed_tracing_complete = false
       end
 
+      #
+      # Accepts block, records trace and by default returns trace data from `Tracing.tracingComplete` event as output.
+      #
+      # @param [String, nil] path
+      #   Save data on the disk.
+      #
+      # @param [:binary, :base64] encoding
+      #   Encode output as Base64 or plain text.
+      #
+      # @param [Float, nil] timeout
+      #   Wait until file streaming finishes in the specified time or raise
+      #   error.
+      #
+      # @param [Boolean] screenshots
+      #   capture screenshots in the trace.
+      #
+      # @param [Hash{String => Object}] trace_config
+      #   config for [trace](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#type-TraceConfig),
+      #   for categories see [getCategories](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#method-getCategories),
+      #   only one trace config can be active at a time per browser.
+      #
+      # @return [String, true]
+      #   The trace data from the `Tracing.tracingComplete` event.
+      #   When `path` is specified returns `true` and stores trace data into
+      #   file.
+      #
       def record(path: nil, encoding: :binary, timeout: nil, trace_config: nil, screenshots: false)
         @path = path
         @encoding = encoding

data/lib/ferrum/page.rb

@@ -35,7 +35,7 @@ module Ferrum
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
                 current_url current_title url title body doctype content=
-                execution_id evaluate evaluate_on evaluate_async execute evaluate_func
+                execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
 
     include Animation
@@ -43,23 +43,47 @@ module Ferrum
     include Frames
     include Stream
 
-    attr_accessor :referrer
-    attr_reader :target_id, :browser,
-                :headers, :cookies, :network,
-                :mouse, :keyboard, :event,
-                :tracing
+    attr_accessor :referrer, :timeout
+    attr_reader :target_id, :browser, :event, :tracing
 
-    def initialize(target_id, browser)
-      @frames = {}
+    # Mouse object.
+    #
+    # @return [Mouse]
+    attr_reader :mouse
+
+    # Keyboard object.
+    #
+    # @return [Keyboard]
+    attr_reader :keyboard
+
+    # Network object.
+    #
+    # @return [Network]
+    attr_reader :network
+
+    # Headers object.
+    #
+    # @return [Headers]
+    attr_reader :headers
+
+    # Cookie store.
+    #
+    # @return [Cookies]
+    attr_reader :cookies
+
+    def initialize(target_id, browser, proxy: nil)
+      @frames = Concurrent::Map.new
       @main_frame = Frame.new(nil, self)
       @browser = browser
       @target_id = target_id
+      @timeout = @browser.timeout
       @event = Event.new.tap(&:set)
+      self.proxy = proxy
 
-      host = @browser.process.host
-      port = @browser.process.port
-      ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
-      @client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
+      @client = Browser::Client.new(ws_url, self,
+                                    logger: @browser.options.logger,
+                                    ws_max_receive_size: @browser.options.ws_max_receive_size,
+                                    id_starts_with: 1000)
 
       @mouse = Mouse.new(self)
       @keyboard = Keyboard.new(self)
@@ -72,30 +96,31 @@ module Ferrum
       prepare_page
     end
 
-    def timeout
-      @browser.timeout
-    end
-
     def context
       @browser.contexts.find_by(target_id: target_id)
     end
 
+    #
+    # Navigates the page to a URL.
+    #
+    # @param [String, nil] url
+    #   The URL to navigate to. The url should include scheme unless you set
+    #   `{Browser#base_url = url}` when configuring driver.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #
     def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
       response = command("Page.navigate", wait: GOTO_WAIT, **options)
-      # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
-      if %w[net::ERR_NAME_NOT_RESOLVED
-            net::ERR_NAME_RESOLUTION_FAILED
-            net::ERR_INTERNET_DISCONNECTED
-            net::ERR_CONNECTION_TIMED_OUT].include?(response["errorText"])
-        raise StatusError, options[:url]
-      end
+      error_text = response["errorText"]
+      raise StatusError.new(options[:url], "Request to #{options[:url]} failed (#{error_text})") if error_text
 
       response["frameId"]
     rescue TimeoutError
-      if @browser.pending_connection_errors
-        pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
+      if @browser.options.pending_connection_errors
+        pendings = network.traffic.select(&:pending?).map(&:url).compact
         raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
       end
     end
@@ -120,34 +145,87 @@ module Ferrum
       command("Emulation.setDeviceMetricsOverride", slowmoable: true,
                                                     width: width,
                                                     height: height,
-                                                    deviceScaleFactor: 1,
-                                                    mobile: false,
-                                                    fitWindow: false)
+                                                    deviceScaleFactor: 0,
+                                                    mobile: false)
     end
 
+    #
+    # The current position of the browser window.
+    #
+    # @return [(Integer, Integer)]
+    #   The left, top coordinates of the browser window.
+    #
+    # @example
+    #   browser.position # => [10, 20]
+    #
     def position
       @browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
     end
 
+    #
+    # Sets the position of the browser window.
+    #
+    # @param [Hash{Symbol => Object}] options
+    #
+    # @option options [Integer] :left
+    #   The number of pixels from the left-hand side of the screen.
+    #
+    # @option options [Integer] :top
+    #   The number of pixels from the top of the screen.
+    #
+    # @example
+    #   browser.position = { left: 10, top: 20 }
+    #
     def position=(options)
       @browser.command("Browser.setWindowBounds",
                        windowId: window_id,
                        bounds: { left: options[:left], top: options[:top] })
     end
 
+    #
+    # Reloads the current page.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.refresh
+    #
     def refresh
       command("Page.reload", wait: timeout, slowmoable: true)
     end
     alias reload refresh
 
+    #
+    # Stop all navigations and loading pending resources on the page.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.stop
+    #
     def stop
       command("Page.stopLoading", slowmoable: true)
     end
 
+    #
+    # Navigates to the previous URL in the browser's history.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.at_xpath("//a").click
+    #   browser.back
+    #
     def back
       history_navigate(delta: -1)
     end
 
+    #
+    # Navigates to the next URL in the browser's history.
+    #
+    # @example
+    #   browser.go_to("https://github.com/")
+    #   browser.at_xpath("//a").click
+    #   browser.back
+    #   browser.forward
+    #
     def forward
       history_navigate(delta: 1)
     end
@@ -158,6 +236,20 @@ module Ferrum
       @event.set
     end
 
+    #
+    # Enables/disables CSP bypass.
+    #
+    # @param [Boolean] enabled
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   browser.bypass_csp # => true
+    #   browser.go_to("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
+    #   browser.refresh
+    #   browser.add_script_tag(content: "window.__injected = 42")
+    #   browser.evaluate("window.__injected") # => 42
+    #
     def bypass_csp(enabled: true)
       command("Page.setBypassCSP", enabled: enabled)
       enabled
@@ -173,16 +265,16 @@ module Ferrum
 
     def command(method, wait: 0, slowmoable: false, **params)
       iteration = @event.reset if wait.positive?
-      sleep(@browser.slowmo) if slowmoable && @browser.slowmo.positive?
+      sleep(@browser.options.slowmo) if slowmoable && @browser.options.slowmo.positive?
       result = @client.command(method, params)
 
       if wait.positive?
-        @event.wait(wait)
         # Wait a bit after command and check if iteration has
         # changed which means there was some network event for
         # the main frame and it started to load new content.
+        @event.wait(wait)
         if iteration != @event.iteration
-          set = @event.wait(@browser.timeout)
+          set = @event.wait(timeout)
           raise TimeoutError unless set
         end
       end
@@ -218,19 +310,31 @@ module Ferrum
       @client.subscribed?(event)
     end
 
+    def use_proxy?
+      @proxy_host && @proxy_port
+    end
+
+    def use_authorized_proxy?
+      use_proxy? && @proxy_user && @proxy_password
+    end
+
+    def document_node_id
+      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    end
+
     private
 
     def subscribe
       frames_subscribe
       network.subscribe
 
-      if @browser.logger
+      if @browser.options.logger
         on("Runtime.consoleAPICalled") do |params|
-          params["args"].each { |r| @browser.logger.puts(r["value"]) }
+          params["args"].each { |r| @browser.options.logger.puts(r["value"]) }
         end
       end
 
-      if @browser.js_errors
+      if @browser.options.js_errors
         on("Runtime.exceptionThrown") do |params|
           # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
           Thread.main.raise JavaScriptError.new(
@@ -243,7 +347,7 @@ module Ferrum
       on(:dialog) do |dialog, _index, total|
         if total == 1
           warn "Dialog was shown but you didn't provide `on(:dialog)` callback, accepting it by default. " \
-               "Please take a look at https://github.com/rubycdp/ferrum#dialog"
+               "Please take a look at https://github.com/rubycdp/ferrum#dialogs"
           dialog.accept
         end
       end
@@ -257,21 +361,22 @@ module Ferrum
       command("Log.enable")
       command("Network.enable")
 
-      if @browser.proxy_options && @browser.proxy_options[:user] && @browser.proxy_options[:password]
-        auth_options = @browser.proxy_options.slice(:user, :password)
-        network.authorize(type: :proxy, **auth_options) do |request, _index, _total|
+      if use_authorized_proxy?
+        network.authorize(user: @proxy_user,
+                          password: @proxy_password,
+                          type: :proxy) do |request, _index, _total|
           request.continue
         end
       end
 
-      if @browser.options[:save_path]
-        unless Pathname.new(@browser.options[:save_path]).absolute?
+      if @browser.options.save_path
+        unless Pathname.new(@browser.options.save_path).absolute?
           raise Error, "supply absolute path for `:save_path` option"
         end
 
         @browser.command("Browser.setDownloadBehavior",
                          browserContextId: context.id,
-                         downloadPath: browser.options[:save_path],
+                         downloadPath: @browser.options.save_path,
                          behavior: "allow", eventsEnabled: true)
       end
 
@@ -285,7 +390,8 @@ module Ferrum
       resize(width: width, height: height)
 
       response = command("Page.getNavigationHistory")
-      return unless response.dig("entries", 0, "transitionType") != "typed"
+      transition_type = response.dig("entries", 0, "transitionType")
+      return if transition_type == "auto_toplevel"
 
       # If we create page by clicking links, submitting forms and so on it
       # opens a new window for which `frameStoppedLoading` event never
@@ -303,7 +409,7 @@ module Ferrum
         # We also evaluate script just in case because
         # `Page.addScriptToEvaluateOnNewDocument` doesn't work in popups.
         command("Runtime.evaluate", expression: extension,
-                                    contextId: execution_id,
+                                    executionContextId: execution_id!,
                                     returnByValue: true)
       end
     end
@@ -333,8 +439,15 @@ module Ferrum
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
     end
 
-    def document_node_id
-      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    def ws_url
+      "ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
+    end
+
+    def proxy=(options)
+      @proxy_host = options&.[](:host) || @browser.options.proxy&.[](:host)
+      @proxy_port = options&.[](:port) || @browser.options.proxy&.[](:port)
+      @proxy_user = options&.[](:user) || @browser.options.proxy&.[](:user)
+      @proxy_password = options&.[](:password) || @browser.options.proxy&.[](:password)
     end
   end
 end