ferrum 0.14 → 0.16

data/lib/ferrum/page.rb

@@ -1,50 +1,43 @@
 # frozen_string_literal: true
 
 require "forwardable"
+require "pathname"
 require "ferrum/mouse"
 require "ferrum/keyboard"
 require "ferrum/headers"
 require "ferrum/cookies"
 require "ferrum/dialog"
 require "ferrum/network"
+require "ferrum/downloads"
 require "ferrum/page/frames"
 require "ferrum/page/screenshot"
 require "ferrum/page/animation"
 require "ferrum/page/tracing"
 require "ferrum/page/stream"
-require "ferrum/browser/client"
 
 module Ferrum
   class Page
     GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
 
-    class Event < Concurrent::Event
-      def iteration
-        synchronize { @iteration }
-      end
-
-      def reset
-        synchronize do
-          @iteration += 1
-          @set = false if @set
-          @iteration
-        end
-      end
-    end
-
     extend Forwardable
     delegate %i[at_css at_xpath css xpath
                 current_url current_title url title body doctype content=
                 execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag] => :main_frame
+    delegate %i[base_url default_user_agent timeout timeout=] => :@options
 
     include Animation
     include Screenshot
     include Frames
     include Stream
 
-    attr_accessor :referrer, :timeout
-    attr_reader :target_id, :browser, :event, :tracing
+    attr_accessor :referrer
+    attr_reader :context_id, :target_id, :event, :tracing
+
+    # Client connection.
+    #
+    # @return [Client]
+    attr_reader :client
 
     # Mouse object.
     #
@@ -71,55 +64,56 @@ module Ferrum
     # @return [Cookies]
     attr_reader :cookies
 
-    def initialize(target_id, browser, proxy: nil)
+    # Downloads object.
+    #
+    # @return [Downloads]
+    attr_reader :downloads
+
+    def initialize(client, context_id:, target_id:, proxy: nil)
+      @client = client
+      @context_id = context_id
+      @target_id = target_id
+      @options = client.options
+
       @frames = Concurrent::Map.new
       @main_frame = Frame.new(nil, self)
-      @browser = browser
-      @target_id = target_id
-      @timeout = @browser.timeout
-      @event = Event.new.tap(&:set)
+      @event = Utils::Event.new.tap(&:set)
       self.proxy = proxy
 
-      @client = Browser::Client.new(ws_url, self,
-                                    logger: @browser.options.logger,
-                                    ws_max_receive_size: @browser.options.ws_max_receive_size,
-                                    id_starts_with: 1000)
-
       @mouse = Mouse.new(self)
       @keyboard = Keyboard.new(self)
       @headers = Headers.new(self)
       @cookies = Cookies.new(self)
       @network = Network.new(self)
       @tracing = Tracing.new(self)
+      @downloads = Downloads.new(self)
 
       subscribe
       prepare_page
     end
 
-    def context
-      @browser.contexts.find_by(target_id: target_id)
-    end
-
     #
     # Navigates the page to a URL.
     #
     # @param [String, nil] url
     #   The URL to navigate to. The url should include scheme unless you set
-    #   `{Browser#base_url = url}` when configuring driver.
+    #   `{Browser#base_url = url}` when configuring.
     #
     # @example
-    #   browser.go_to("https://github.com/")
+    #   page.go_to("https://github.com/")
     #
     def go_to(url = nil)
       options = { url: combine_url!(url) }
       options.merge!(referrer: referrer) if referrer
       response = command("Page.navigate", wait: GOTO_WAIT, **options)
-      error_text = response["errorText"]
-      raise StatusError.new(options[:url], "Request to #{options[:url]} failed (#{error_text})") if error_text
+      error_text = response["errorText"] # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
+      if error_text && error_text != "net::ERR_ABORTED" # Request aborted due to user action or download
+        raise StatusError.new(options[:url], "Request to #{options[:url]} failed (#{error_text})")
+      end
 
       response["frameId"]
     rescue TimeoutError
-      if @browser.options.pending_connection_errors
+      if @options.pending_connection_errors
         pendings = network.traffic.select(&:pending?).map(&:url).compact
         raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
       end
@@ -129,41 +123,76 @@ module Ferrum
 
     def close
       @headers.clear
-      @browser.command("Target.closeTarget", targetId: @target_id)
-      @client.close
+      client.command("Target.closeTarget", async: true, targetId: @target_id)
+      close_connection
+
+      true
+    end
+
+    def close_connection
+      client&.close
+    end
+
+    #
+    # Overrides device screen dimensions and emulates viewport according to parameters
+    #
+    # Read more [here](https://chromedevtools.github.io/devtools-protocol/tot/Emulation/#method-setDeviceMetricsOverride).
+    #
+    # @param [Integer] width width value in pixels. 0 disables the override
+    #
+    # @param [Integer] height height value in pixels. 0 disables the override
+    #
+    # @param [Float] scale_factor device scale factor value. 0 disables the override
+    #
+    # @param [Boolean] mobile whether to emulate mobile device
+    #
+    def set_viewport(width:, height:, scale_factor: 0, mobile: false)
+      command(
+        "Emulation.setDeviceMetricsOverride",
+        slowmoable: true,
+        width: width,
+        height: height,
+        deviceScaleFactor: scale_factor,
+        mobile: mobile
+      )
     end
 
     def resize(width: nil, height: nil, fullscreen: false)
       if fullscreen
         width, height = document_size
-        set_window_bounds(windowState: "fullscreen")
+        self.window_bounds = { window_state: "fullscreen" }
       else
-        set_window_bounds(windowState: "normal")
-        set_window_bounds(width: width, height: height)
+        self.window_bounds = { window_state: "normal" }
+        self.window_bounds = { width: width, height: height }
       end
 
-      command("Emulation.setDeviceMetricsOverride", slowmoable: true,
-                                                    width: width,
-                                                    height: height,
-                                                    deviceScaleFactor: 0,
-                                                    mobile: false)
+      set_viewport(width: width, height: height)
+    end
+
+    #
+    # Disables JavaScript execution from the HTML source for the page.
+    #
+    # This doesn't prevent users evaluate JavaScript with Ferrum.
+    #
+    def disable_javascript
+      command("Emulation.setScriptExecutionDisabled", value: true)
     end
 
     #
-    # The current position of the browser window.
+    # The current position of the window.
     #
     # @return [(Integer, Integer)]
-    #   The left, top coordinates of the browser window.
+    #   The left, top coordinates of the window.
     #
     # @example
-    #   browser.position # => [10, 20]
+    #   page.position # => [10, 20]
     #
     def position
-      @browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
+      window_bounds.values_at("left", "top")
     end
 
     #
-    # Sets the position of the browser window.
+    # Sets the position of the window.
     #
     # @param [Hash{Symbol => Object}] options
     #
@@ -174,20 +203,72 @@ module Ferrum
     #   The number of pixels from the top of the screen.
     #
     # @example
-    #   browser.position = { left: 10, top: 20 }
+    #   page.position = { left: 10, top: 20 }
     #
     def position=(options)
-      @browser.command("Browser.setWindowBounds",
-                       windowId: window_id,
-                       bounds: { left: options[:left], top: options[:top] })
+      self.window_bounds = { left: options[:left], top: options[:top] }
+    end
+
+    # Sets the position of the window.
+    #
+    # @param [Hash{Symbol => Object}] bounds
+    #
+    # @option options [Integer] :left
+    #   The number of pixels from the left-hand side of the screen.
+    #
+    # @option options [Integer] :top
+    #   The number of pixels from the top of the screen.
+    #
+    # @option options [Integer] :width
+    #   The window width in pixels.
+    #
+    # @option options [Integer] :height
+    #   The window height in pixels.
+    #
+    # @option options [String] :window_state
+    #   The window state. Default to normal. Allowed Values: normal, minimized, maximized, fullscreen
+    #
+    # @example
+    #   page.window_bounds = { left: 10, top: 20, width: 1024, height: 768, window_state: "normal" }
+    #
+    def window_bounds=(bounds)
+      options = bounds.dup
+      window_state = options.delete(:window_state)
+      bounds = { windowState: window_state, **options }.compact
+
+      client.command("Browser.setWindowBounds", windowId: window_id, bounds: bounds)
+    end
+
+    #
+    # Current window bounds.
+    #
+    # @return [Hash{String => (Integer, String)}]
+    #
+    # @example
+    #   page.window_bounds # => { "left": 0, "top": 1286, "width": 10, "height": 10, "windowState": "normal" }
+    #
+    def window_bounds
+      client.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds")
+    end
+
+    #
+    # Current window id.
+    #
+    # @return [Integer]
+    #
+    # @example
+    #   page.window_id # => 1
+    #
+    def window_id
+      client.command("Browser.getWindowForTarget", targetId: target_id)["windowId"]
     end
 
     #
     # Reloads the current page.
     #
     # @example
-    #   browser.go_to("https://github.com/")
-    #   browser.refresh
+    #   page.go_to("https://github.com/")
+    #   page.refresh
     #
     def refresh
       command("Page.reload", wait: timeout, slowmoable: true)
@@ -198,41 +279,41 @@ module Ferrum
     # Stop all navigations and loading pending resources on the page.
     #
     # @example
-    #   browser.go_to("https://github.com/")
-    #   browser.stop
+    #   page.go_to("https://github.com/")
+    #   page.stop
     #
     def stop
       command("Page.stopLoading", slowmoable: true)
     end
 
     #
-    # Navigates to the previous URL in the browser's history.
+    # Navigates to the previous URL in the history.
     #
     # @example
-    #   browser.go_to("https://github.com/")
-    #   browser.at_xpath("//a").click
-    #   browser.back
+    #   page.go_to("https://github.com/")
+    #   page.at_xpath("//a").click
+    #   page.back
     #
     def back
       history_navigate(delta: -1)
     end
 
     #
-    # Navigates to the next URL in the browser's history.
+    # Navigates to the next URL in the history.
     #
     # @example
-    #   browser.go_to("https://github.com/")
-    #   browser.at_xpath("//a").click
-    #   browser.back
-    #   browser.forward
+    #   page.go_to("https://github.com/")
+    #   page.at_xpath("//a").click
+    #   page.back
+    #   page.forward
     #
     def forward
       history_navigate(delta: 1)
     end
 
-    def wait_for_reload(sec = 1)
+    def wait_for_reload(timeout = 1)
       @event.reset if @event.set?
-      @event.wait(sec)
+      @event.wait(timeout)
       @event.set
     end
 
@@ -244,29 +325,35 @@ module Ferrum
     # @return [Boolean]
     #
     # @example
-    #   browser.bypass_csp # => true
-    #   browser.go_to("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
-    #   browser.refresh
-    #   browser.add_script_tag(content: "window.__injected = 42")
-    #   browser.evaluate("window.__injected") # => 42
+    #   page.bypass_csp # => true
+    #   page.go_to("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
+    #   page.refresh
+    #   page.add_script_tag(content: "window.__injected = 42")
+    #   page.evaluate("window.__injected") # => 42
     #
     def bypass_csp(enabled: true)
       command("Page.setBypassCSP", enabled: enabled)
       enabled
     end
 
-    def window_id
-      @browser.command("Browser.getWindowForTarget", targetId: @target_id)["windowId"]
-    end
-
-    def set_window_bounds(bounds = {})
-      @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: bounds)
+    #
+    # Activates (focuses) the target for the given page.
+    # When you have multiple tabs you work with, and you need to switch a given one.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   page.activate # => true
+    #
+    def activate
+      command("Target.activateTarget", targetId: target_id)
+      true
     end
 
     def command(method, wait: 0, slowmoable: false, **params)
       iteration = @event.reset if wait.positive?
-      sleep(@browser.options.slowmo) if slowmoable && @browser.options.slowmo.positive?
-      result = @client.command(method, params)
+      sleep(@options.slowmo) if slowmoable && @options.slowmo.positive?
+      result = client.command(method, **params)
 
       if wait.positive?
         # Wait a bit after command and check if iteration has
@@ -284,30 +371,30 @@ module Ferrum
     def on(name, &block)
       case name
       when :dialog
-        @client.on("Page.javascriptDialogOpening") do |params, index, total|
+        client.on("Page.javascriptDialogOpening") do |params, index, total|
           dialog = Dialog.new(self, params)
           block.call(dialog, index, total)
         end
       when :request
-        @client.on("Fetch.requestPaused") do |params, index, total|
-          request = Network::InterceptedRequest.new(self, params)
+        client.on("Fetch.requestPaused") do |params, index, total|
+          request = Network::InterceptedRequest.new(client, params)
           exchange = network.select(request.network_id).last
           exchange ||= network.build_exchange(request.network_id)
           exchange.intercepted_request = request
           block.call(request, index, total)
         end
       when :auth
-        @client.on("Fetch.authRequired") do |params, index, total|
+        client.on("Fetch.authRequired") do |params, index, total|
           request = Network::AuthRequest.new(self, params)
           block.call(request, index, total)
         end
       else
-        @client.on(name, &block)
+        client.on(name, &block)
       end
     end
 
     def subscribed?(event)
-      @client.subscribed?(event)
+      client.subscribed?(event)
     end
 
     def use_proxy?
@@ -318,7 +405,9 @@ module Ferrum
       use_proxy? && @proxy_user && @proxy_password
     end
 
-    def document_node_id
+    def document_node_id(async: false)
+      return client.command("DOM.getDocument", async: true, depth: 0) if async
+
       command("DOM.getDocument", depth: 0).dig("root", "nodeId")
     end
 
@@ -327,14 +416,15 @@ module Ferrum
     def subscribe
       frames_subscribe
       network.subscribe
+      downloads.subscribe
 
-      if @browser.options.logger
+      if @options.logger
         on("Runtime.consoleAPICalled") do |params|
-          params["args"].each { |r| @browser.options.logger.puts(r["value"]) }
+          params["args"].each { |r| @options.logger.puts(r["value"]) }
         end
       end
 
-      if @browser.options.js_errors
+      if @options.js_errors
         on("Runtime.exceptionThrown") do |params|
           # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
           Thread.main.raise JavaScriptError.new(
@@ -369,26 +459,14 @@ module Ferrum
         end
       end
 
-      if @browser.options.save_path
-        unless Pathname.new(@browser.options.save_path).absolute?
-          raise Error, "supply absolute path for `:save_path` option"
-        end
-
-        @browser.command("Browser.setDownloadBehavior",
-                         browserContextId: context.id,
-                         downloadPath: @browser.options.save_path,
-                         behavior: "allow", eventsEnabled: true)
-      end
+      downloads.set_behavior(save_path: @options.save_path) if @options.save_path
 
-      @browser.extensions.each do |extension|
+      @options.extensions.each do |extension|
         command("Page.addScriptToEvaluateOnNewDocument", source: extension)
       end
 
       inject_extensions
 
-      width, height = @browser.window_size
-      resize(width: width, height: height)
-
       response = command("Page.getNavigationHistory")
       transition_type = response.dig("entries", 0, "transitionType")
       return if transition_type == "auto_toplevel"
@@ -402,7 +480,7 @@ module Ferrum
     end
 
     def inject_extensions
-      @browser.extensions.each do |extension|
+      @options.extensions.each do |extension|
         # https://github.com/GoogleChrome/puppeteer/issues/1443
         # https://github.com/ChromeDevTools/devtools-protocol/issues/77
         # https://github.com/cyrus-and/chrome-remote-interface/issues/319
@@ -432,22 +510,18 @@ module Ferrum
       url = Addressable::URI.parse(url_or_path)
       nil_or_relative = url.nil? || url.relative?
 
-      if nil_or_relative && !@browser.base_url
+      if nil_or_relative && !@options.base_url
         raise "Set :base_url browser's option or use absolute url in `go_to`, you passed: #{url_or_path}"
       end
 
-      (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
-    end
-
-    def ws_url
-      "ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
+      (nil_or_relative ? @options.base_url.join(url.to_s) : url).to_s
     end
 
     def proxy=(options)
-      @proxy_host = options&.[](:host) || @browser.options.proxy&.[](:host)
-      @proxy_port = options&.[](:port) || @browser.options.proxy&.[](:port)
-      @proxy_user = options&.[](:user) || @browser.options.proxy&.[](:user)
-      @proxy_password = options&.[](:password) || @browser.options.proxy&.[](:password)
+      @proxy_host = options&.[](:host) || @options.proxy&.[](:host)
+      @proxy_port = options&.[](:port) || @options.proxy&.[](:port)
+      @proxy_user = options&.[](:user) || @options.proxy&.[](:user)
+      @proxy_password = options&.[](:password) || @options.proxy&.[](:password)
     end
   end
 end

data/lib/ferrum/proxy.rb

@@ -52,7 +52,7 @@ module Ferrum
     end
 
     def stop
-      @file&.unlink
+      @file&.close(true)
       @server.shutdown
     end
 

data/lib/ferrum/target.rb

@@ -8,17 +8,21 @@ module Ferrum
     # where we enhance page class and build page ourselves.
     attr_writer :page
 
-    def initialize(browser, params = nil)
+    attr_reader :session_id, :options
+
+    def initialize(browser_client, session_id = nil, params = nil)
       @page = nil
-      @browser = browser
+      @session_id = session_id
       @params = params
+      @browser_client = browser_client
+      @options = browser_client.options
     end
 
     def update(params)
-      @params = params
+      @params.merge!(params)
     end
 
-    def attached?
+    def connected?
       !!@page
     end
 
@@ -26,9 +30,13 @@ module Ferrum
       @page ||= build_page
     end
 
+    def client
+      @client ||= build_client
+    end
+
     def build_page(**options)
       maybe_sleep_if_new_window
-      Page.new(id, @browser, **options)
+      Page.new(client, context_id: context_id, target_id: id, **options)
     end
 
     def id
@@ -63,5 +71,17 @@ module Ferrum
       # Dirty hack because new window doesn't have events at all
       sleep(NEW_WINDOW_WAIT) if window?
     end
+
+    private
+
+    def build_client
+      return @browser_client.session(session_id) if options.flatten
+
+      Client.new(ws_url, options)
+    end
+
+    def ws_url
+      @browser_client.ws_url.merge(path: "/devtools/page/#{id}")
+    end
   end
 end

data/lib/ferrum/utils/elapsed_time.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "concurrent-ruby"
-
 module Ferrum
   module Utils
     module ElapsedTime

data/lib/ferrum/utils/event.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Ferrum
+  module Utils
+    class Event < Concurrent::Event
+      def iteration
+        synchronize { @iteration }
+      end
+
+      def reset
+        synchronize do
+          @iteration += 1
+          @set = false if @set
+          @iteration
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/utils/platform.rb

@@ -20,6 +20,10 @@ module Ferrum
         RbConfig::CONFIG["host_os"] =~ /darwin/
       end
 
+      def mac_arm?
+        mac? && RbConfig::CONFIG["host_cpu"] =~ /arm/
+      end
+
       def mri?
         defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
       end

data/lib/ferrum/utils/thread.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Ferrum
+  module Utils
+    module Thread
+      module_function
+
+      def spawn(abort_on_exception: true)
+        ::Thread.new(abort_on_exception) do |whether_abort_on_exception|
+          ::Thread.current.abort_on_exception = whether_abort_on_exception
+          ::Thread.current.report_on_exception = true if ::Thread.current.respond_to?(:report_on_exception=)
+
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.14"
+  VERSION = "0.16"
 end

data/lib/ferrum.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "concurrent-ruby"
+require "ferrum/utils/event"
+require "ferrum/utils/thread"
 require "ferrum/utils/platform"
 require "ferrum/utils/elapsed_time"
 require "ferrum/utils/attempt"