ferrum 0.9 → 0.10

data/lib/ferrum.rb

@@ -10,14 +10,20 @@ module Ferrum
   class NotImplementedError < Error; end
 
   class StatusError < Error
+    def initialize(url, message = nil)
+      super(message || "Request to #{url} failed to reach server, check DNS and server status")
+    end
+  end
+
+  class PendingConnectionsError < StatusError
+    attr_reader :pendings
+
     def initialize(url, pendings = [])
-      message = if pendings.empty?
-                  "Request to #{url} failed to reach server, check DNS and/or server status"
-                else
-                  "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
-                end
+      @pendings = pendings
 
-      super(message)
+      message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
+
+      super(url, message)
     end
   end
 
@@ -37,8 +43,11 @@ module Ferrum
   end
 
   class ProcessTimeoutError < Error
-    def initialize(timeout)
-      super("Browser did not produce websocket url within #{timeout} seconds")
+    attr_reader :output
+
+    def initialize(timeout, output)
+      @output = output
+      super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
     end
   end
 

data/lib/ferrum/browser.rb

@@ -17,19 +17,19 @@ module Ferrum
     extend Forwardable
     delegate %i[default_context] => :contexts
     delegate %i[targets create_target create_page page pages windows] => :default_context
-    delegate %i[goto back forward refresh reload stop wait_for_reload
+    delegate %i[go_to back forward refresh reload stop wait_for_reload
                 at_css at_xpath css xpath current_url current_title url title
                 body doctype set_content
                 headers cookies network
                 mouse keyboard
-                screenshot pdf viewport_size
+                screenshot pdf mhtml viewport_size
                 frames frame_by main_frame
-                evaluate evaluate_on evaluate_async execute
+                evaluate evaluate_on evaluate_async execute evaluate_func
                 add_script_tag add_style_tag bypass_csp
-                on] => :page
+                on goto] => :page
     delegate %i[default_user_agent] => :process
 
-    attr_reader :client, :process, :contexts, :logger, :js_errors,
+    attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
                 :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
     attr_writer :timeout
 
@@ -44,6 +44,7 @@ module Ferrum
       @logger, @timeout, @ws_max_receive_size =
         @options.values_at(:logger, :timeout, :ws_max_receive_size)
       @js_errors = @options.fetch(:js_errors, false)
+      @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
       @slowmo = @options[:slowmo].to_f
 
       if @options.key?(:base_url)

data/lib/ferrum/browser/client.rb

@@ -58,6 +58,10 @@ module Ferrum
         end
       end
 
+      def subscribed?(event)
+        [@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
+      end
+
       def close
         @ws.close
         # Give a thread some time to handle a tail of messages

data/lib/ferrum/browser/command.rb

@@ -4,7 +4,7 @@ module Ferrum
   class Browser
     class Command
       NOT_FOUND = "Could not find an executable for the browser. Try to make " \
-                  "it available on the PATH or set environment varible for " \
+                  "it available on the PATH or set environment variable for " \
                   "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
 
       # Currently only these browsers support CDP:

data/lib/ferrum/browser/process.rb

@@ -16,7 +16,7 @@ module Ferrum
     class Process
       KILL_TIMEOUT = 2
       WAIT_KILLED = 0.05
-      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 2).to_i
+      PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
 
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
@@ -64,11 +64,13 @@ module Ferrum
           return
         end
 
+        @pid = @xvfb = @user_data_dir = nil
         @logger = options[:logger]
         @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
 
-        tmpdir = Dir.mktmpdir
+        tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
         ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
+        @user_data_dir = tmpdir
         @command = Command.build(options, tmpdir)
       end
 
@@ -144,7 +146,7 @@ module Ferrum
 
         unless ws_url
           @logger.puts(output) if @logger
-          raise ProcessTimeoutError.new(timeout)
+          raise ProcessTimeoutError.new(timeout, output)
         end
       end
 

data/lib/ferrum/browser/subscriber.rb

@@ -21,6 +21,10 @@ module Ferrum
         true
       end
 
+      def subscribed?(event)
+        @on.key?(event)
+      end
+
       def call(message)
         method, params = message.values_at("method", "params")
         total = @on[method].size

data/lib/ferrum/context.rb

@@ -42,9 +42,9 @@ module Ferrum
     end
 
     def create_target
-      target_id = @browser.command("Target.createTarget",
-                                   browserContextId: @id,
-                                   url: "about:blank")["targetId"]
+      @browser.command("Target.createTarget",
+                       browserContextId: @id,
+                       url: "about:blank")
       target = @pendings.take(@browser.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
       @targets[target.id] = target

data/lib/ferrum/frame.rb

@@ -11,6 +11,7 @@ module Ferrum
     attr_accessor :id, :name
 
     def initialize(id, page, parent_id = nil)
+      @execution_id = nil
       @id, @page, @parent_id = id, page, parent_id
     end
 

data/lib/ferrum/frame/dom.rb

@@ -37,54 +37,54 @@ module Ferrum
       end
 
       def xpath(selector, within: nil)
-        code = <<~JS
-          let selector = arguments[0];
-          let within = arguments[1] || document;
-          let results = [];
+        expr = <<~JS
+          function(selector, within) {
+            let results = [];
+            within ||= document
 
-          let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-          for (let i = 0; i < xpath.snapshotLength; i++) {
-            results.push(xpath.snapshotItem(i));
-          }
+            let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+            for (let i = 0; i < xpath.snapshotLength; i++) {
+              results.push(xpath.snapshotItem(i));
+            }
 
-          arguments[2](results);
+            return results;
+          }
         JS
 
-        evaluate_async(code, @page.timeout, selector, within)
+        evaluate_func(expr, selector, within)
       end
 
       def at_xpath(selector, within: nil)
-        code = <<~JS
-          let selector = arguments[0];
-          let within = arguments[1] || document;
-          let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-          let result = xpath.snapshotItem(0);
-          arguments[2](result);
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+            return xpath.snapshotItem(0);
+          }
         JS
-
-        evaluate_async(code, @page.timeout, selector, within)
+        evaluate_func(expr, selector, within)
       end
 
       def css(selector, within: nil)
-        code = <<~JS
-          let selector = arguments[0];
-          let within = arguments[1] || document;
-          let results = within.querySelectorAll(selector);
-          arguments[2](results);
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            return Array.from(within.querySelectorAll(selector));
+          }
         JS
 
-        evaluate_async(code, @page.timeout, selector, within)
+        evaluate_func(expr, selector, within)
       end
 
       def at_css(selector, within: nil)
-        code = <<~JS
-          let selector = arguments[0];
-          let within = arguments[1] || document;
-          let result = within.querySelector(selector);
-          arguments[2](result);
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            return within.querySelector(selector);
+          }
         JS
 
-        evaluate_async(code, @page.timeout, selector, within)
+        evaluate_func(expr, selector, within)
       end
     end
   end

data/lib/ferrum/frame/runtime.rb

@@ -3,36 +3,19 @@
 require "singleton"
 
 module Ferrum
+  class CyclicObject
+    include Singleton
+
+    def inspect
+      %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
+    end
+  end
+
   class Frame
     module Runtime
       INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
       INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
 
-      EXECUTE_OPTIONS = {
-        returnByValue: true,
-        functionDeclaration: %(function() { %s })
-      }.freeze
-      DEFAULT_OPTIONS = {
-        functionDeclaration: %(function() { return %s })
-      }.freeze
-      EVALUATE_ASYNC_OPTIONS = {
-        awaitPromise: true,
-        functionDeclaration: %(
-          function() {
-           return new Promise((__resolve, __reject) => {
-             try {
-               arguments[arguments.length] = r => __resolve(r);
-               arguments.length = arguments.length + 1;
-               setTimeout(() => __reject(new Error("timed out promise")), %s);
-               %s
-             } catch(error) {
-               __reject(error);
-             }
-           });
-          }
-        )
-      }.freeze
-
       SCRIPT_SRC_TAG = <<~JS
         const script = document.createElement("script");
         script.src = arguments[0];
@@ -63,37 +46,45 @@ module Ferrum
       JS
 
       def evaluate(expression, *args)
-        call(*args, expression: expression)
+        expression = "function() { return %s }" % expression
+        call(expression: expression, arguments: args)
       end
 
-      def evaluate_async(expression, wait_time, *args)
-        call(*args, expression: expression, wait_time: wait_time * 1000, **EVALUATE_ASYNC_OPTIONS)
+      def evaluate_async(expression, wait, *args)
+        template = <<~JS
+          function() {
+            return new Promise((__f, __r) => {
+              try {
+                arguments[arguments.length] = r => __f(r);
+                arguments.length = arguments.length + 1;
+                setTimeout(() => __r(new Error("timed out promise")), %s);
+                %s
+              } catch(error) {
+                __r(error);
+              }
+            });
+          }
+        JS
+
+        expression = template % [wait * 1000, expression]
+        call(expression: expression, arguments: args, awaitPromise: true)
       end
 
       def execute(expression, *args)
-        call(*args, expression: expression, handle: false, **EXECUTE_OPTIONS)
+        expression = "function() { %s }" % expression
+        call(expression: expression, arguments: args, handle: false, returnByValue: true)
         true
       end
 
-      def evaluate_on(node:, expression:, by_value: true, wait: 0)
-        errors = [NodeNotFoundError, NoExecutionContextError]
-        attempts, sleep = INTERMITTENT_ATTEMPTS, INTERMITTENT_SLEEP
-
-        Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
-          response = @page.command("DOM.resolveNode", nodeId: node.node_id)
-          object_id = response.dig("object", "objectId")
-          options = DEFAULT_OPTIONS.merge(objectId: object_id)
-          options[:functionDeclaration] = options[:functionDeclaration] % expression
-          options.merge!(returnByValue: by_value)
-
-          response = @page.command("Runtime.callFunctionOn",
-                                   wait: wait, slowmoable: true,
-                                   **options)
-          handle_error(response)
-          response = response["result"]
+      def evaluate_func(expression, *args, on: nil)
+        call(expression: expression, arguments: args, on: on)
+      end
 
-          by_value ? response.dig("value") : handle_response(response)
-        end
+      def evaluate_on(node:, expression:, by_value: true, wait: 0)
+        options = { handle: true }
+        expression = "function() { return %s }" % expression
+        options = { handle: false, returnByValue: true } if by_value
+        call(expression: expression, on: node, wait: wait, **options)
       end
 
       def add_script_tag(url: nil, path: nil, content: nil, type: "text/javascript")
@@ -126,27 +117,29 @@ module Ferrum
 
       private
 
-      def call(*args, expression:, wait_time: nil, handle: true, **options)
+      def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
         errors = [NodeNotFoundError, NoExecutionContextError]
         attempts, sleep = INTERMITTENT_ATTEMPTS, INTERMITTENT_SLEEP
 
         Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
-          arguments = prepare_args(args)
-          params = DEFAULT_OPTIONS.merge(options)
-          expression = [wait_time, expression] if wait_time
-          params[:functionDeclaration] = params[:functionDeclaration] % expression
-          params = params.merge(arguments: arguments)
-          unless params[:executionContextId]
-            params = params.merge(executionContextId: execution_id)
+          if on
+            response = @page.command("DOM.resolveNode", nodeId: on.node_id)
+            object_id = response.dig("object", "objectId")
+            options.merge!(objectId: object_id)
+          else
+            options.merge!(executionContextId: execution_id)
           end
 
+          options.merge!(functionDeclaration: expression,
+                         arguments: prepare_args(arguments))
+
           response = @page.command("Runtime.callFunctionOn",
-                                   slowmoable: true,
-                                   **params)
+                                   wait: wait, slowmoable: true,
+                                   **options)
           handle_error(response)
           response = response["result"]
 
-          handle ? handle_response(response) : response
+          handle ? handle_response(response) : response.dig("value")
         end
       end
 
@@ -239,15 +232,13 @@ module Ferrum
                 return false;
               }
 
-              const seen = [];
-              function detectCycle(obj) {
+              function detectCycle(obj, seen) {
                 if (typeof obj === "object") {
                   if (seen.indexOf(obj) !== -1) {
                     return true;
                   }
-                  seen.push(obj);
                   for (let key in obj) {
-                    if (obj.hasOwnProperty(key) && detectCycle(obj[key])) {
+                    if (obj.hasOwnProperty(key) && detectCycle(obj[key], seen.concat([obj]))) {
                       return true;
                     }
                   }
@@ -256,7 +247,7 @@ module Ferrum
                 return false;
               }
 
-              return detectCycle(this);
+              return detectCycle(this, []);
             }
           JS
         )
@@ -267,12 +258,4 @@ module Ferrum
       end
     end
   end
-
-  class CyclicObject
-    include Singleton
-
-    def inspect
-      %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
-    end
-  end
 end

data/lib/ferrum/network.rb

@@ -83,19 +83,21 @@ module Ferrum
       @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
     end
 
-    def authorize(user:, password:, type: :server)
+    def authorize(user:, password:, type: :server, &block)
       unless AUTHORIZE_TYPE.include?(type)
         raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
       end
 
+      if !block_given? && !@page.subscribed?("Fetch.requestPaused")
+        raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
+      end
+
       @authorized_ids ||= {}
       @authorized_ids[type] ||= []
 
       intercept
 
-      @page.on(:request) do |request|
-        request.continue
-      end
+      @page.on(:request, &block)
 
       @page.on(:auth) do |request, index, total|
         if request.auth_challenge?(type)
@@ -157,12 +159,27 @@ module Ferrum
         end
       end
 
+      @page.on("Network.loadingFailed") do |params|
+        exchange = select(params["requestId"]).last
+        exchange.error ||= Network::Error.new
+
+        exchange.error.id = params["requestId"]
+        exchange.error.type = params["type"]
+        exchange.error.error_text = params["errorText"]
+        exchange.error.monotonic_time = params["timestamp"]
+        exchange.error.canceled = params["canceled"]
+      end
+
       @page.on("Log.entryAdded") do |params|
         entry = params["entry"] || {}
         if entry["source"] == "network" && entry["level"] == "error"
           exchange = select(entry["networkRequestId"]).last
-          error = Network::Error.new(entry)
-          exchange.error = error
+          exchange.error ||= Network::Error.new
+
+          exchange.error.id = entry["networkRequestId"]
+          exchange.error.url = entry["url"]
+          exchange.error.description = entry["text"]
+          exchange.error.timestamp = entry["timestamp"]
         end
       end
     end