ferrum 0.6 → 0.9

data/lib/ferrum/browser/subscriber.rb

@@ -7,9 +7,13 @@ module Ferrum
     class Subscriber
       include Concurrent::Async
 
+      def self.build(size)
+        (0..size).map { new }
+      end
+
       def initialize
         super
-        @on = Hash.new { |h, k| h[k] = [] }
+        @on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
       end
 
       def on(event, &block)

data/lib/ferrum/browser/web_socket.rb

@@ -8,24 +8,32 @@ module Ferrum
   class Browser
     class WebSocket
       WEBSOCKET_BUG_SLEEP = 0.01
+      SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
 
       attr_reader :url, :messages
 
-      def initialize(url, logger)
+      def initialize(url, max_receive_size, logger)
         @url      = url
         @logger   = logger
         uri       = URI.parse(@url)
         @sock     = TCPSocket.new(uri.host, uri.port)
-        @driver   = ::WebSocket::Driver.client(self)
+        max_receive_size ||= ::WebSocket::Driver::MAX_LENGTH
+        @driver   = ::WebSocket::Driver.client(self, max_length: max_receive_size)
         @messages = Queue.new
 
+        if SKIP_LOGGING_SCREENSHOTS
+          @screenshot_commands = Concurrent::Hash.new
+        end
+
         @driver.on(:open,    &method(:on_open))
         @driver.on(:message, &method(:on_message))
         @driver.on(:close,   &method(:on_close))
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
+          if Thread.current.respond_to?(:report_on_exception=)
+            Thread.current.report_on_exception = true
+          end
 
           begin
             while data = @sock.readpartial(512)
@@ -47,7 +55,14 @@ module Ferrum
       def on_message(event)
         data = JSON.parse(event.data)
         @messages.push(data)
-        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{event.data}\n")
+
+        output = event.data
+        if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data["id"]]
+          @screenshot_commands.delete(data["id"])
+          output.sub!(/{"data":"(.*)"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
+        end
+
+        @logger&.puts("    ◀ #{Ferrum.elapsed_time} #{output}\n")
       end
 
       def on_close(_event)
@@ -56,6 +71,10 @@ module Ferrum
       end
 
       def send_message(data)
+        if SKIP_LOGGING_SCREENSHOTS
+          @screenshot_commands[data[:id]] = true
+        end
+
         json = data.to_json
         @driver.text(json)
         @logger&.puts("\n\n▶ #{Ferrum.elapsed_time} #{json}")

data/lib/ferrum/browser/xvfb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    class Xvfb
+      NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
+                  "it with your package manager".freeze
+
+      def self.start(*args)
+        new(*args).tap(&:start)
+      end
+
+      def self.xvfb_path
+        Cliver.detect("Xvfb")
+      end
+
+      attr_reader :screen_size, :display_id, :pid
+
+      def initialize(options)
+        @path = self.class.xvfb_path
+        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+
+        @screen_size = options.fetch(:window_size, [1024, 768]).join("x") + "x24"
+        @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
+      end
+
+      def start
+        @pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
+        ::Process.detach(@pid)
+      end
+
+      def to_env
+        { "DISPLAY" => ":#{display_id}" }
+      end
+    end
+  end
+end

data/lib/ferrum/cookies.rb

@@ -23,6 +23,10 @@ module Ferrum
         @attributes["path"]
       end
 
+      def samesite
+        @attributes["sameSite"]
+      end
+
       def size
         @attributes["size"]
       end
@@ -65,6 +69,9 @@ module Ferrum
       cookie[:value]  ||= value
       cookie[:domain] ||= default_domain
 
+      cookie[:httpOnly] = cookie.delete(:httponly) if cookie.key?(:httponly)
+      cookie[:sameSite] = cookie.delete(:samesite) if cookie.key?(:samesite)
+
       expires = cookie.delete(:expires).to_i
       cookie[:expires] = expires if expires > 0
 

data/lib/ferrum/dialog.rb

@@ -14,11 +14,11 @@ module Ferrum
       options = { accept: true }
       response = prompt_text || default_prompt
       options.merge!(promptText: response) if response
-      @page.command("Page.handleJavaScriptDialog", **options)
+      @page.command("Page.handleJavaScriptDialog", slowmoable: true, **options)
     end
 
     def dismiss
-      @page.command("Page.handleJavaScriptDialog", accept: false)
+      @page.command("Page.handleJavaScriptDialog", slowmoable: true, accept: false)
     end
 
     def match?(regexp)

data/lib/ferrum/frame.rb

@@ -7,9 +7,8 @@ module Ferrum
   class Frame
     include DOM, Runtime
 
-    attr_reader :id, :page, :parent_id, :state
-    attr_writer :execution_id
-    attr_accessor :name
+    attr_reader :page, :parent_id, :state
+    attr_accessor :id, :name
 
     def initialize(id, page, parent_id = nil)
       @id, @page, @parent_id = id, page, parent_id
@@ -18,8 +17,6 @@ module Ferrum
     # Can be one of:
     # * started_loading
     # * navigated
-    # * scheduled_navigation
-    # * cleared_scheduled_navigation
     # * stopped_loading
     def state=(value)
       @state = value
@@ -37,6 +34,19 @@ module Ferrum
       @parent_id.nil?
     end
 
+    def set_content(html)
+      evaluate_async(%(
+        document.open();
+        document.write(arguments[0]);
+        document.close();
+        arguments[1](true);
+      ), @page.timeout, html)
+    end
+
+    def execution_id?(execution_id)
+      @execution_id == execution_id
+    end
+
     def execution_id
       raise NoExecutionContextError unless @execution_id
       @execution_id
@@ -45,6 +55,14 @@ module Ferrum
       @page.event.wait(@page.timeout) ? retry : raise
     end
 
+    def set_execution_id(value)
+      @execution_id ||= value
+    end
+
+    def reset_execution_id
+      @execution_id = nil
+    end
+
     def inspect
       %(#<#{self.class} @id=#{@id.inspect} @parent_id=#{@parent_id.inspect} @name=#{@name.inspect} @state=#{@state.inspect} @execution_id=#{@execution_id.inspect}>)
     end

data/lib/ferrum/frame/dom.rb

@@ -29,65 +29,62 @@ module Ferrum
       end
 
       def doctype
-        evaluate("new XMLSerializer().serializeToString(document.doctype)")
+        evaluate("document.doctype && new XMLSerializer().serializeToString(document.doctype)")
       end
 
       def body
         evaluate("document.documentElement.outerHTML")
       end
 
-      def at_xpath(selector, within: nil)
-        xpath(selector, within: within).first
-      end
-
-      # FIXME: Check within
       def xpath(selector, within: nil)
-        evaluate_async(%(
-          try {
-            let selector = arguments[0];
-            let within = arguments[1] || document;
-            let results = [];
+        code = <<~JS
+          let selector = arguments[0];
+          let within = arguments[1] || document;
+          let results = [];
 
-            let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-            for (let i = 0; i < xpath.snapshotLength; i++) {
-              results.push(xpath.snapshotItem(i));
-            }
+          let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+          for (let i = 0; i < xpath.snapshotLength; i++) {
+            results.push(xpath.snapshotItem(i));
+          }
 
-            arguments[2](results);
-          } catch (error) {
-            // DOMException.INVALID_EXPRESSION_ERR is undefined, using pure code
-            if (error.code == DOMException.SYNTAX_ERR || error.code == 51) {
-              throw "Invalid Selector";
-            } else {
-              throw error;
-            }
-          }), @page.timeout, selector, within)
+          arguments[2](results);
+        JS
+
+        evaluate_async(code, @page.timeout, selector, within)
       end
 
-      # FIXME css doesn't work for a frame w/o execution_id
-      def css(selector, within: nil)
-        node_id = within&.node_id || @page.document_id
+      def at_xpath(selector, within: nil)
+        code = <<~JS
+          let selector = arguments[0];
+          let within = arguments[1] || document;
+          let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+          let result = xpath.snapshotItem(0);
+          arguments[2](result);
+        JS
 
-        ids = @page.command("DOM.querySelectorAll",
-                            nodeId: node_id,
-                            selector: selector)["nodeIds"]
-        ids.map { |id| build_node(id) }.compact
+        evaluate_async(code, @page.timeout, selector, within)
       end
 
-      def at_css(selector, within: nil)
-        node_id = within&.node_id || @page.document_id
+      def css(selector, within: nil)
+        code = <<~JS
+          let selector = arguments[0];
+          let within = arguments[1] || document;
+          let results = within.querySelectorAll(selector);
+          arguments[2](results);
+        JS
 
-        id = @page.command("DOM.querySelector",
-                     nodeId: node_id,
-                     selector: selector)["nodeId"]
-        build_node(id)
+        evaluate_async(code, @page.timeout, selector, within)
       end
 
-      private
+      def at_css(selector, within: nil)
+        code = <<~JS
+          let selector = arguments[0];
+          let within = arguments[1] || document;
+          let result = within.querySelector(selector);
+          arguments[2](result);
+        JS
 
-      def build_node(node_id)
-        description = @page.command("DOM.describeNode", nodeId: node_id)
-        Node.new(self, @page.target_id, node_id, description["node"])
+        evaluate_async(code, @page.timeout, selector, within)
       end
     end
   end

data/lib/ferrum/frame/runtime.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "singleton"
+
 module Ferrum
   class Frame
     module Runtime
@@ -85,8 +87,10 @@ module Ferrum
           options.merge!(returnByValue: by_value)
 
           response = @page.command("Runtime.callFunctionOn",
-                             wait: wait, **options)["result"]
-                            .tap { |r| handle_error(r) }
+                                   wait: wait, slowmoable: true,
+                                   **options)
+          handle_error(response)
+          response = response["result"]
 
           by_value ? response.dig("value") : handle_response(response)
         end
@@ -137,14 +141,18 @@ module Ferrum
           end
 
           response = @page.command("Runtime.callFunctionOn",
-                             **params)["result"].tap { |r| handle_error(r) }
+                                   slowmoable: true,
+                                   **params)
+          handle_error(response)
+          response = response["result"]
 
           handle ? handle_response(response) : response
         end
       end
 
       # FIXME: We should have a central place to handle all type of errors
-      def handle_error(result)
+      def handle_error(response)
+        result = response["result"]
         return if result["subtype"] != "error"
 
         case result["description"]
@@ -209,7 +217,7 @@ module Ferrum
 
       def reduce_props(object_id, to)
         if cyclic?(object_id).dig("result", "value")
-          return "(cyclic structure)"
+          return to.is_a?(Array) ? [cyclic_object] : cyclic_object
         else
           props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)
           props["result"].reduce(to) do |memo, prop|
@@ -220,38 +228,51 @@ module Ferrum
       end
 
       def cyclic?(object_id)
-        @page.command("Runtime.callFunctionOn",
-                objectId: object_id,
-                returnByValue: true,
-                functionDeclaration: <<~JS
-                  function() {
-                    if (Array.isArray(this) &&
-                        this.every(e => e instanceof Node)) {
-                      return false;
-                    }
+        @page.command(
+          "Runtime.callFunctionOn",
+          objectId: object_id,
+          returnByValue: true,
+          functionDeclaration: <<~JS
+            function() {
+              if (Array.isArray(this) &&
+                  this.every(e => e instanceof Node)) {
+                return false;
+              }
 
-                    const seen = [];
-                    function detectCycle(obj) {
-                      if (typeof obj === 'object') {
-                        if (seen.indexOf(obj) !== -1) {
-                          return true;
-                        }
-                        seen.push(obj);
-                        for (let key in obj) {
-                          if (obj.hasOwnProperty(key) && detectCycle(obj[key])) {
-                            return true;
-                          }
-                        }
-                      }
-
-                      return false;
+              const seen = [];
+              function detectCycle(obj) {
+                if (typeof obj === "object") {
+                  if (seen.indexOf(obj) !== -1) {
+                    return true;
+                  }
+                  seen.push(obj);
+                  for (let key in obj) {
+                    if (obj.hasOwnProperty(key) && detectCycle(obj[key])) {
+                      return true;
                     }
-
-                    return detectCycle(this);
                   }
-                JS
-               )
+                }
+
+                return false;
+              }
+
+              return detectCycle(this);
+            }
+          JS
+        )
       end
+
+      def cyclic_object
+        CyclicObject.instance
+      end
+    end
+  end
+
+  class CyclicObject
+    include Singleton
+
+    def inspect
+      %(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
     end
   end
 end

data/lib/ferrum/headers.rb

@@ -40,7 +40,7 @@ module Ferrum
 
     def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
       options = Hash.new
-      options[:userAgent] = user_agent if user_agent
+      options[:userAgent] = user_agent || @page.browser.default_user_agent
       options[:acceptLanguage] = accept_language if accept_language
       options[:platform] if platform