ferrum 0.6.1 → 0.10

data/lib/ferrum/browser/subscriber.rb

@@ -7,9 +7,13 @@ module Ferrum
     class Subscriber
       include Concurrent::Async
 
+      def self.build(size)
+        (0..size).map { new }
+      end
+
       def initialize
         super
-        @on = Hash.new { |h, k| h[k] = [] }
+        @on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
       end
 
       def on(event, &block)
@@ -17,6 +21,10 @@ module Ferrum
         true
       end
 
+      def subscribed?(event)
+        @on.key?(event)
+      end
+
       def call(message)
         method, params = message.values_at("method", "params")
         total = @on[method].size

data/lib/ferrum/browser/web_socket.rb

@@ -8,24 +8,32 @@ module Ferrum
   class Browser
     class WebSocket
       WEBSOCKET_BUG_SLEEP = 0.01
+      SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
 
       attr_reader :url, :messages
 
-      def initialize(url, logger)
+      def initialize(url, max_receive_size, logger)
         @url      = url
         @logger   = logger
         uri       = URI.parse(@url)
         @sock     = TCPSocket.new(uri.host, uri.port)
-        @driver   = ::WebSocket::Driver.client(self)
+        max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
+        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
         @messages = Queue.new
 
+        if SKIP_LOGGING_SCREENSHOTS
+          @screenshot_commands = Concurrent::Hash.new
+        end
+
         @driver.on(:open,    &method(:on_open))
         @driver.on(:message, &method(:on_message))
         @driver.on(:close,   &method(:on_close))
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
+          if Thread.current.respond_to?(:report_on_exception=)
+            Thread.current.report_on_exception = true
+          end
 
           begin
             while data = @sock.readpartial(512)
@@ -47,7 +55,14 @@ module Ferrum
       def on_message(event)
         data = JSON.parse(event.data)
         @messages.push(data)
-        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{event.data}\n")
+
+        output = event.data
+        if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data["id"]]
+          @screenshot_commands.delete(data["id"])
+          output.sub!(/{"data":"(.*)"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
+        end
+
+        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{output}\n")
       end
 
       def on_close(_event)
@@ -56,6 +71,10 @@ module Ferrum
       end
 
       def send_message(data)
+        if SKIP_LOGGING_SCREENSHOTS
+          @screenshot_commands[data[:id]] = true
+        end
+
         json = data.to_json
         @driver.text(json)
         @logger&.puts("\n\n▶ #{Ferrum.elapsed_time} #{json}")

data/lib/ferrum/browser/xvfb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Xvfb
+      NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
+                  "it with your package manager".freeze
+
+      def self.start(*args)
+        new(*args).tap(&:start)
+      end
+
+      def self.xvfb_path
+        Cliver.detect("Xvfb")
+      end
+
+      attr_reader :screen_size, :display_id, :pid
+
+      def initialize(options)
+        @path = self.class.xvfb_path
+        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+
+        @screen_size = options.fetch(:window_size, [1024, 768]).join("x") + "x24"
+        @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
+      end
+
+      def start
+        @pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
+        ::Process.detach(@pid)
+      end
+
+      def to_env
+        { "DISPLAY" => ":#{display_id}" }
+      end
+    end
+  end
+end

data/lib/ferrum/context.rb

@@ -42,9 +42,9 @@ module Ferrum
     end
 
     def create_target
-      target_id = @browser.command("Target.createTarget",
-                                   browserContextId: @id,
-                                   url: "about:blank")["targetId"]
+      @browser.command("Target.createTarget",
+                       browserContextId: @id,
+                       url: "about:blank")
       target = @pendings.take(@browser.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
       @targets[target.id] = target

data/lib/ferrum/cookies.rb

@@ -23,6 +23,10 @@ module Ferrum
         @attributes["path"]
       end
 
+      def samesite
+        @attributes["sameSite"]
+      end
+
       def size
         @attributes["size"]
       end
@@ -65,6 +69,9 @@ module Ferrum
       cookie[:value]  ||= value
       cookie[:domain] ||= default_domain
 
+      cookie[:httpOnly] = cookie.delete(:httponly) if cookie.key?(:httponly)
+      cookie[:sameSite] = cookie.delete(:samesite) if cookie.key?(:samesite)
+
       expires = cookie.delete(:expires).to_i
       cookie[:expires] = expires if expires > 0
 

data/lib/ferrum/dialog.rb

@@ -14,11 +14,11 @@ module Ferrum
       options = { accept: true }
       response = prompt_text || default_prompt
       options.merge!(promptText: response) if response
-      @page.command("Page.handleJavaScriptDialog", **options)
+      @page.command("Page.handleJavaScriptDialog", slowmoable: true, **options)
     end
 
     def dismiss
-      @page.command("Page.handleJavaScriptDialog", accept: false)
+      @page.command("Page.handleJavaScriptDialog", slowmoable: true, accept: false)
     end
 
     def match?(regexp)

data/lib/ferrum/frame.rb

@@ -7,19 +7,17 @@ module Ferrum
   class Frame
     include DOM, Runtime
 
-    attr_reader :id, :page, :parent_id, :state
-    attr_writer :execution_id
-    attr_accessor :name
+    attr_reader :page, :parent_id, :state
+    attr_accessor :id, :name
 
     def initialize(id, page, parent_id = nil)
+      @execution_id = nil
       @id, @page, @parent_id = id, page, parent_id
     end
 
     # Can be one of:
     # * started_loading
     # * navigated
-    # * scheduled_navigation
-    # * cleared_scheduled_navigation
     # * stopped_loading
     def state=(value)
       @state = value
@@ -37,6 +35,15 @@ module Ferrum
       @parent_id.nil?
     end
 
+    def set_content(html)
+      evaluate_async(%(
+        document.open();
+        document.write(arguments[0]);
+        document.close();
+        arguments[1](true);
+      ), @page.timeout, html)
+    end
+
     def execution_id?(execution_id)
       @execution_id == execution_id
     end
@@ -49,6 +56,14 @@ module Ferrum
       @page.event.wait(@page.timeout) ? retry : raise
     end
 
+    def set_execution_id(value)
+      @execution_id ||= value
+    end
+
+    def reset_execution_id
+      @execution_id = nil
+    end
+
     def inspect
       %(#<#{self.class} @id=#{@id.inspect} @parent_id=#{@parent_id.inspect} @name=#{@name.inspect} @state=#{@state.inspect} @execution_id=#{@execution_id.inspect}>)
     end

data/lib/ferrum/frame/dom.rb

@@ -29,65 +29,62 @@ module Ferrum
       end
 
       def doctype
-        evaluate("new XMLSerializer().serializeToString(document.doctype)")
+        evaluate("document.doctype && new XMLSerializer().serializeToString(document.doctype)")
       end
 
       def body
         evaluate("document.documentElement.outerHTML")
       end
 
-      def at_xpath(selector, within: nil)
-        xpath(selector, within: within).first
-      end
-
-      # FIXME: Check within
       def xpath(selector, within: nil)
-        evaluate_async(%(
-          try {
-            let selector = arguments[0];
-            let within = arguments[1] || document;
+        expr = <<~JS
+          function(selector, within) {
             let results = [];
+            within ||= document
 
             let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
             for (let i = 0; i < xpath.snapshotLength; i++) {
               results.push(xpath.snapshotItem(i));
             }
 
-            arguments[2](results);
-          } catch (error) {
-            // DOMException.INVALID_EXPRESSION_ERR is undefined, using pure code
-            if (error.code == DOMException.SYNTAX_ERR || error.code == 51) {
-              throw "Invalid Selector";
-            } else {
-              throw error;
-            }
-          }), @page.timeout, selector, within)
-      end
+            return results;
+          }
+        JS
 
-      # FIXME css doesn't work for a frame w/o execution_id
-      def css(selector, within: nil)
-        node_id = within&.node_id || @page.document_id
+        evaluate_func(expr, selector, within)
+      end
 
-        ids = @page.command("DOM.querySelectorAll",
-                            nodeId: node_id,
-                            selector: selector)["nodeIds"]
-        ids.map { |id| build_node(id) }.compact
+      def at_xpath(selector, within: nil)
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+            return xpath.snapshotItem(0);
+          }
+        JS
+        evaluate_func(expr, selector, within)
       end
 
-      def at_css(selector, within: nil)
-        node_id = within&.node_id || @page.document_id
+      def css(selector, within: nil)
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            return Array.from(within.querySelectorAll(selector));
+          }
+        JS
 
-        id = @page.command("DOM.querySelector",
-                     nodeId: node_id,
-                     selector: selector)["nodeId"]
-        build_node(id)
+        evaluate_func(expr, selector, within)
       end
 
-      private
+      def at_css(selector, within: nil)
+        expr = <<~JS
+          function(selector, within) {
+            within ||= document
+            return within.querySelector(selector);
+          }
+        JS
 
-      def build_node(node_id)
-        description = @page.command("DOM.describeNode", nodeId: node_id)
-        Node.new(self, @page.target_id, node_id, description["node"])
+        evaluate_func(expr, selector, within)
       end
     end
   end

data/lib/ferrum/frame/runtime.rb

@@ -1,36 +1,21 @@
 # frozen_string_literal: true
 
+require "singleton"
+
 module Ferrum
+  class CyclicObject
+    include Singleton
+
+    def inspect
+      %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
+    end
+  end
+
   class Frame
     module Runtime
       INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
       INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
 
-      EXECUTE_OPTIONS = {
-        returnByValue: true,
-        functionDeclaration: %(function() { %s })
-      }.freeze
-      DEFAULT_OPTIONS = {
-        functionDeclaration: %(function() { return %s })
-      }.freeze
-      EVALUATE_ASYNC_OPTIONS = {
-        awaitPromise: true,
-        functionDeclaration: %(
-          function() {
-           return new Promise((__resolve, __reject) => {
-             try {
-               arguments[arguments.length] = r => __resolve(r);
-               arguments.length = arguments.length + 1;
-               setTimeout(() => __reject(new Error("timed out promise")), %s);
-               %s
-             } catch(error) {
-               __reject(error);
-             }
-           });
-          }
-        )
-      }.freeze
-
       SCRIPT_SRC_TAG = <<~JS
         const script = document.createElement("script");
         script.src = arguments[0];
@@ -61,35 +46,45 @@ module Ferrum
       JS
 
       def evaluate(expression, *args)
-        call(*args, expression: expression)
+        expression = "function() { return %s }" % expression
+        call(expression: expression, arguments: args)
       end
 
-      def evaluate_async(expression, wait_time, *args)
-        call(*args, expression: expression, wait_time: wait_time * 1000, **EVALUATE_ASYNC_OPTIONS)
+      def evaluate_async(expression, wait, *args)
+        template = <<~JS
+          function() {
+            return new Promise((__f, __r) => {
+              try {
+                arguments[arguments.length] = r => __f(r);
+                arguments.length = arguments.length + 1;
+                setTimeout(() => __r(new Error("timed out promise")), %s);
+                %s
+              } catch(error) {
+                __r(error);
+              }
+            });
+          }
+        JS
+
+        expression = template % [wait * 1000, expression]
+        call(expression: expression, arguments: args, awaitPromise: true)
       end
 
       def execute(expression, *args)
-        call(*args, expression: expression, handle: false, **EXECUTE_OPTIONS)
+        expression = "function() { %s }" % expression
+        call(expression: expression, arguments: args, handle: false, returnByValue: true)
         true
       end
 
-      def evaluate_on(node:, expression:, by_value: true, wait: 0)
-        errors = [NodeNotFoundError, NoExecutionContextError]
-        attempts, sleep = INTERMITTENT_ATTEMPTS, INTERMITTENT_SLEEP
-
-        Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
-          response = @page.command("DOM.resolveNode", nodeId: node.node_id)
-          object_id = response.dig("object", "objectId")
-          options = DEFAULT_OPTIONS.merge(objectId: object_id)
-          options[:functionDeclaration] = options[:functionDeclaration] % expression
-          options.merge!(returnByValue: by_value)
-
-          response = @page.command("Runtime.callFunctionOn",
-                             wait: wait, **options)["result"]
-                            .tap { |r| handle_error(r) }
+      def evaluate_func(expression, *args, on: nil)
+        call(expression: expression, arguments: args, on: on)
+      end
 
-          by_value ? response.dig("value") : handle_response(response)
-        end
+      def evaluate_on(node:, expression:, by_value: true, wait: 0)
+        options = { handle: true }
+        expression = "function() { return %s }" % expression
+        options = { handle: false, returnByValue: true } if by_value
+        call(expression: expression, on: node, wait: wait, **options)
       end
 
       def add_script_tag(url: nil, path: nil, content: nil, type: "text/javascript")
@@ -122,29 +117,35 @@ module Ferrum
 
       private
 
-      def call(*args, expression:, wait_time: nil, handle: true, **options)
+      def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
         errors = [NodeNotFoundError, NoExecutionContextError]
         attempts, sleep = INTERMITTENT_ATTEMPTS, INTERMITTENT_SLEEP
 
         Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
-          arguments = prepare_args(args)
-          params = DEFAULT_OPTIONS.merge(options)
-          expression = [wait_time, expression] if wait_time
-          params[:functionDeclaration] = params[:functionDeclaration] % expression
-          params = params.merge(arguments: arguments)
-          unless params[:executionContextId]
-            params = params.merge(executionContextId: execution_id)
+          if on
+            response = @page.command("DOM.resolveNode", nodeId: on.node_id)
+            object_id = response.dig("object", "objectId")
+            options.merge!(objectId: object_id)
+          else
+            options.merge!(executionContextId: execution_id)
           end
 
+          options.merge!(functionDeclaration: expression,
+                         arguments: prepare_args(arguments))
+
           response = @page.command("Runtime.callFunctionOn",
-                             **params)["result"].tap { |r| handle_error(r) }
+                                   wait: wait, slowmoable: true,
+                                   **options)
+          handle_error(response)
+          response = response["result"]
 
-          handle ? handle_response(response) : response
+          handle ? handle_response(response) : response.dig("value")
         end
       end
 
       # FIXME: We should have a central place to handle all type of errors
-      def handle_error(result)
+      def handle_error(response)
+        result = response["result"]
         return if result["subtype"] != "error"
 
         case result["description"]
@@ -209,7 +210,7 @@ module Ferrum
 
       def reduce_props(object_id, to)
         if cyclic?(object_id).dig("result", "value")
-          return "(cyclic structure)"
+          return to.is_a?(Array) ? [cyclic_object] : cyclic_object
         else
           props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)
           props["result"].reduce(to) do |memo, prop|
@@ -220,37 +221,40 @@ module Ferrum
       end
 
       def cyclic?(object_id)
-        @page.command("Runtime.callFunctionOn",
-                objectId: object_id,
-                returnByValue: true,
-                functionDeclaration: <<~JS
-                  function() {
-                    if (Array.isArray(this) &&
-                        this.every(e => e instanceof Node)) {
-                      return false;
-                    }
+        @page.command(
+          "Runtime.callFunctionOn",
+          objectId: object_id,
+          returnByValue: true,
+          functionDeclaration: <<~JS
+            function() {
+              if (Array.isArray(this) &&
+                  this.every(e => e instanceof Node)) {
+                return false;
+              }
 
-                    const seen = [];
-                    function detectCycle(obj) {
-                      if (typeof obj === 'object') {
-                        if (seen.indexOf(obj) !== -1) {
-                          return true;
-                        }
-                        seen.push(obj);
-                        for (let key in obj) {
-                          if (obj.hasOwnProperty(key) && detectCycle(obj[key])) {
-                            return true;
-                          }
-                        }
-                      }
-
-                      return false;
+              function detectCycle(obj, seen) {
+                if (typeof obj === "object") {
+                  if (seen.indexOf(obj) !== -1) {
+                    return true;
+                  }
+                  for (let key in obj) {
+                    if (obj.hasOwnProperty(key) && detectCycle(obj[key], seen.concat([obj]))) {
+                      return true;
                     }
-
-                    return detectCycle(this);
                   }
-                JS
-               )
+                }
+
+                return false;
+              }
+
+              return detectCycle(this, []);
+            }
+          JS
+        )
+      end
+
+      def cyclic_object
+        CyclicObject.instance
       end
     end
   end