ferrum 0.13 → 0.15

data/lib/ferrum/client.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "ferrum/client/subscriber"
+require "ferrum/client/web_socket"
+
+module Ferrum
+  class SessionClient
+    attr_reader :client, :session_id
+
+    def self.event_name(event, session_id)
+      [event, session_id].compact.join("_")
+    end
+
+    def initialize(client, session_id)
+      @client = client
+      @session_id = session_id
+    end
+
+    def command(method, async: false, **params)
+      message = build_message(method, params)
+      @client.send_message(message, async: async)
+    end
+
+    def on(event, &block)
+      @client.on(event_name(event), &block)
+    end
+
+    def subscribed?(event)
+      @client.subscribed?(event_name(event))
+    end
+
+    def respond_to_missing?(name, include_private)
+      @client.respond_to?(name, include_private)
+    end
+
+    def method_missing(name, ...)
+      @client.send(name, ...)
+    end
+
+    def close
+      @client.subscriber.clear(session_id: session_id)
+    end
+
+    private
+
+    def build_message(method, params)
+      @client.build_message(method, params).merge(sessionId: session_id)
+    end
+
+    def event_name(event)
+      self.class.event_name(event, session_id)
+    end
+  end
+
+  class Client
+    extend Forwardable
+    delegate %i[timeout timeout=] => :options
+
+    attr_reader :ws_url, :options, :subscriber
+
+    def initialize(ws_url, options)
+      @command_id = 0
+      @ws_url = ws_url
+      @options = options
+      @pendings = Concurrent::Hash.new
+      @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
+      @subscriber = Subscriber.new
+
+      start
+    end
+
+    def command(method, async: false, **params)
+      message = build_message(method, params)
+      send_message(message, async: async)
+    end
+
+    def send_message(message, async:)
+      if async
+        @ws.send_message(message)
+        true
+      else
+        pending = Concurrent::IVar.new
+        @pendings[message[:id]] = pending
+        @ws.send_message(message)
+        data = pending.value!(timeout)
+        @pendings.delete(message[:id])
+
+        raise DeadBrowserError if data.nil? && @ws.messages.closed?
+        raise TimeoutError unless data
+
+        error, response = data.values_at("error", "result")
+        raise_browser_error(error) if error
+        response
+      end
+    end
+
+    def on(event, &block)
+      @subscriber.on(event, &block)
+    end
+
+    def subscribed?(event)
+      @subscriber.subscribed?(event)
+    end
+
+    def session(session_id)
+      SessionClient.new(self, session_id)
+    end
+
+    def close
+      @ws.close
+      # Give a thread some time to handle a tail of messages
+      @pendings.clear
+      @thread.kill unless @thread.join(1)
+      @subscriber.close
+    end
+
+    def inspect
+      "#<#{self.class} " \
+        "@command_id=#{@command_id.inspect} " \
+        "@pendings=#{@pendings.inspect} " \
+        "@ws=#{@ws.inspect}>"
+    end
+
+    def build_message(method, params)
+      { method: method, params: params }.merge(id: next_command_id)
+    end
+
+    private
+
+    def start
+      @thread = Utils::Thread.spawn do
+        loop do
+          message = @ws.messages.pop
+          break unless message
+
+          if message.key?("method")
+            @subscriber << message
+          else
+            @pendings[message["id"]]&.set(message)
+          end
+        end
+      end
+    end
+
+    def next_command_id
+      @command_id += 1
+    end
+
+    def raise_browser_error(error)
+      case error["message"]
+      # Node has disappeared while we were trying to get it
+      when "No node with given id found",
+           "Could not find node with given id",
+           "Inspected target navigated or closed"
+        raise NodeNotFoundError, error
+      # Context is lost, page is reloading
+      when "Cannot find context with specified id"
+        raise NoExecutionContextError, error
+      when "No target with given id found"
+        raise NoSuchPageError
+      when /Could not compute content quads/
+        raise CoordinatesNotFoundError
+      else
+        raise BrowserError, error
+      end
+    end
+  end
+end

data/lib/ferrum/context.rb

@@ -8,9 +8,9 @@ module Ferrum
 
     attr_reader :id, :targets
 
-    def initialize(browser, contexts, id)
+    def initialize(client, contexts, id)
       @id = id
-      @browser = browser
+      @client = client
       @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::MVar.new
@@ -46,33 +46,37 @@ module Ferrum
     end
 
     def create_target
-      @browser.command("Target.createTarget",
-                       browserContextId: @id,
-                       url: "about:blank")
-      target = @pendings.take(@browser.timeout)
+      @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")
+      target = @pendings.take(@client.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
 
-      @targets.put_if_absent(target.id, target)
       target
     end
 
-    def add_target(params)
-      target = Target.new(@browser, params)
-      if target.window?
-        @targets.put_if_absent(target.id, target)
-      else
-        @pendings.put(target, @browser.timeout)
-      end
+    def add_target(params:, session_id: nil)
+      new_target = Target.new(@client, session_id, params)
+      target = @targets.put_if_absent(new_target.id, new_target)
+      target ||= new_target # `put_if_absent` returns nil if added a new value or existing if there was one already
+      @pendings.put(target, @client.timeout) if @pendings.empty?
+      target
     end
 
     def update_target(target_id, params)
-      @targets[target_id].update(params)
+      @targets[target_id]&.update(params)
     end
 
     def delete_target(target_id)
       @targets.delete(target_id)
     end
 
+    def close_targets_connection
+      @targets.each_value do |target|
+        next unless target.connected?
+
+        target.page.close_connection
+      end
+    end
+
     def dispose
       @contexts.dispose(@id)
     end

data/lib/ferrum/contexts.rb

@@ -4,12 +4,15 @@ require "ferrum/context"
 
 module Ferrum
   class Contexts
+    include Enumerable
+
     attr_reader :contexts
 
-    def initialize(browser)
+    def initialize(client)
       @contexts = Concurrent::Map.new
-      @browser = browser
+      @client = client
       subscribe
+      auto_attach
       discover
     end
 
@@ -17,6 +20,16 @@ module Ferrum
       @default_context ||= create
     end
 
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      @contexts.each(&block)
+    end
+
+    def [](id)
+      @contexts[id]
+    end
+
     def find_by(target_id:)
       context = nil
       @contexts.each_value { |c| context = c if c.target?(target_id) }
@@ -24,21 +37,25 @@ module Ferrum
     end
 
     def create(**options)
-      response = @browser.command("Target.createBrowserContext", **options)
+      response = @client.command("Target.createBrowserContext", **options)
       context_id = response["browserContextId"]
-      context = Context.new(@browser, self, context_id)
+      context = Context.new(@client, self, context_id)
       @contexts[context_id] = context
       context
     end
 
     def dispose(context_id)
       context = @contexts[context_id]
-      @browser.command("Target.disposeBrowserContext",
-                       browserContextId: context.id)
+      context.close_targets_connection
+      @client.command("Target.disposeBrowserContext", browserContextId: context.id)
       @contexts.delete(context_id)
       true
     end
 
+    def close_connections
+      @contexts.each_value(&:close_targets_connection)
+    end
+
     def reset
       @default_context = nil
       @contexts.each_key { |id| dispose(id) }
@@ -51,15 +68,26 @@ module Ferrum
     private
 
     def subscribe
-      @browser.client.on("Target.targetCreated") do |params|
+      @client.on("Target.attachedToTarget") do |params|
+        info, session_id = params.values_at("targetInfo", "sessionId")
+        next unless info["type"] == "page"
+
+        context_id = info["browserContextId"]
+        @contexts[context_id]&.add_target(session_id: session_id, params: info)
+        if params["waitingForDebugger"]
+          @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
+        end
+      end
+
+      @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
         next unless info["type"] == "page"
 
         context_id = info["browserContextId"]
-        @contexts[context_id]&.add_target(info)
+        @contexts[context_id]&.add_target(params: info)
       end
 
-      @browser.client.on("Target.targetInfoChanged") do |params|
+      @client.on("Target.targetInfoChanged") do |params|
         info = params["targetInfo"]
         next unless info["type"] == "page"
 
@@ -67,19 +95,25 @@ module Ferrum
         @contexts[context_id]&.update_target(target_id, info)
       end
 
-      @browser.client.on("Target.targetDestroyed") do |params|
+      @client.on("Target.targetDestroyed") do |params|
         context = find_by(target_id: params["targetId"])
         context&.delete_target(params["targetId"])
       end
 
-      @browser.client.on("Target.targetCrashed") do |params|
+      @client.on("Target.targetCrashed") do |params|
         context = find_by(target_id: params["targetId"])
         context&.delete_target(params["targetId"])
       end
     end
 
     def discover
-      @browser.command("Target.setDiscoverTargets", discover: true)
+      @client.command("Target.setDiscoverTargets", discover: true)
+    end
+
+    def auto_attach
+      return unless @client.options.flatten
+
+      @client.command("Target.setAutoAttach", autoAttach: true, waitForDebuggerOnStart: true, flatten: true)
     end
   end
 end

data/lib/ferrum/cookies/cookie.rb

@@ -113,6 +113,38 @@ module Ferrum
         Time.at(attributes["expires"]) if attributes["expires"].positive?
       end
 
+      #
+      # The priority of the cookie.
+      #
+      # @return [String]
+      #
+      def priority
+        @attributes["priority"]
+      end
+
+      #
+      # @return [Boolean]
+      #
+      def sameparty?
+        @attributes["sameParty"]
+      end
+
+      alias same_party? sameparty?
+
+      #
+      # @return [String]
+      #
+      def source_scheme
+        @attributes["sourceScheme"]
+      end
+
+      #
+      # @return [Integer]
+      #
+      def source_port
+        @attributes["sourcePort"]
+      end
+
       #
       # Compares different cookie objects.
       #
@@ -121,6 +153,31 @@ module Ferrum
       def ==(other)
         other.class == self.class && other.attributes == attributes
       end
+
+      #
+      # Converts the cookie back into a raw cookie String.
+      #
+      # @return [String]
+      #   The raw cookie string.
+      #
+      def to_s
+        string = String.new("#{@attributes['name']}=#{@attributes['value']}")
+
+        @attributes.each do |key, value|
+          case key
+          when "name", "value" # no-op
+          when "domain"   then string << "; Domain=#{value}"
+          when "path"     then string << "; Path=#{value}"
+          when "expires"  then string << "; Expires=#{Time.at(value).httpdate}"
+          when "httpOnly" then string << "; httpOnly" if value
+          when "secure"   then string << "; Secure"   if value
+          end
+        end
+
+        string
+      end
+
+      alias to_h attributes
     end
   end
 end

data/lib/ferrum/cookies.rb

@@ -4,10 +4,34 @@ require "ferrum/cookies/cookie"
 
 module Ferrum
   class Cookies
+    include Enumerable
+
     def initialize(page)
       @page = page
     end
 
+    #
+    # Enumerates over all cookies.
+    #
+    # @yield [cookie]
+    #   The given block will be passed each cookie.
+    #
+    # @yieldparam [Cookie] cookie
+    #   A cookie in the browser.
+    #
+    # @return [Enumerator]
+    #   If no block is given, an Enumerator object will be returned.
+    #
+    def each
+      return enum_for(__method__) unless block_given?
+
+      cookies = @page.command("Network.getAllCookies")["cookies"]
+
+      cookies.each do |c|
+        yield Cookie.new(c)
+      end
+    end
+
     #
     # Returns cookies hash.
     #
@@ -22,8 +46,9 @@ module Ferrum
     #   # }
     #
     def all
-      cookies = @page.command("Network.getAllCookies")["cookies"]
-      cookies.to_h { |c| [c["name"], Cookie.new(c)] }
+      each.to_h do |cookie|
+        [cookie.name, cookie]
+      end
     end
 
     #
@@ -44,7 +69,7 @@ module Ferrum
     #   # }>
     #
     def [](name)
-      all[name]
+      find { |cookie| cookie.name == name }
     end
 
     #
@@ -53,23 +78,34 @@ module Ferrum
     # @param [Hash{Symbol => Object}, Cookie] options
     #
     # @option options [String] :name
+    #   The cookie param name.
     #
     # @option options [String] :value
+    #   The cookie param value.
     #
     # @option options [String] :domain
+    #   The domain the cookie belongs to.
     #
     # @option options [String] :path
+    #   The path that the cookie is bound to.
     #
     # @option options [Integer] :expires
+    #   When the cookie will expire.
     #
     # @option options [Integer] :size
+    #   The size of the cookie.
     #
     # @option options [Boolean] :httponly
+    #   Specifies whether the cookie `HttpOnly`.
     #
     # @option options [Boolean] :secure
+    #   Specifies whether the cookie is marked as `Secure`.
     #
     # @option options [String] :samesite
+    #   Specifies whether the cookie is `SameSite`.
     #
+    # @option options [Boolean] :session
+    #   Specifies whether the cookie is a session cookie.
     #
     # @example
     #   browser.cookies.set(name: "stealth", value: "omg", domain: "google.com") # => true
@@ -136,7 +172,7 @@ module Ferrum
     private
 
     def default_domain
-      URI.parse(@page.browser.base_url).host if @page.browser.base_url
+      URI.parse(@page.base_url).host if @page.base_url
     end
   end
 end

data/lib/ferrum/downloads.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Downloads
+    VALID_BEHAVIOR = %i[deny allow allowAndName default].freeze
+
+    def initialize(page)
+      @page = page
+      @event = Utils::Event.new.tap(&:set)
+      @files = {}
+    end
+
+    def files
+      @files.values
+    end
+
+    def wait(timeout = 5)
+      @event.reset
+      yield if block_given?
+      @event.wait(timeout)
+      @event.set
+    end
+
+    def set_behavior(save_path:, behavior: :allow)
+      raise ArgumentError unless VALID_BEHAVIOR.include?(behavior.to_sym)
+      raise Error, "supply absolute path for `:save_path` option" unless Pathname.new(save_path.to_s).absolute?
+
+      @page.command("Browser.setDownloadBehavior",
+                    browserContextId: @page.context_id,
+                    downloadPath: save_path,
+                    behavior: behavior,
+                    eventsEnabled: true)
+    end
+
+    def subscribe
+      subscribe_download_will_begin
+      subscribe_download_progress
+    end
+
+    def subscribe_download_will_begin
+      @page.on("Browser.downloadWillBegin") do |params|
+        @event.reset
+        @files[params["guid"]] = params
+      end
+    end
+
+    def subscribe_download_progress
+      @page.on("Browser.downloadProgress") do |params|
+        @files[params["guid"]].merge!(params)
+
+        case params["state"]
+        when "completed", "canceled"
+          @event.set
+        else
+          @event.reset
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/errors.rb

@@ -6,7 +6,8 @@ module Ferrum
   class NoSuchTargetError   < Error; end
   class NotImplementedError < Error; end
   class BinaryNotFoundError < Error; end
-  class EmptyPathError < Error; end
+  class EmptyPathError      < Error; end
+  class ServerError         < Error; end
 
   class StatusError < Error
     def initialize(url, message = nil)

data/lib/ferrum/frame.rb

@@ -113,6 +113,7 @@ module Ferrum
         document.close();
         arguments[1](true);
       ), @page.timeout, html)
+      @page.document_node_id
     end
     alias set_content content=
 

data/lib/ferrum/headers.rb

@@ -68,7 +68,7 @@ module Ferrum
 
     def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
       options = {}
-      options[:userAgent] = user_agent || @page.browser.default_user_agent
+      options[:userAgent] = user_agent || @page.default_user_agent
       options[:acceptLanguage] = accept_language if accept_language
       options[:platform] if platform
 

data/lib/ferrum/network/exchange.rb

@@ -51,7 +51,7 @@ module Ferrum
       # @return [Boolean]
       #
       def navigation_request?(frame_id)
-        request.type?(:document) && request&.frame_id == frame_id
+        request&.type?(:document) && request&.frame_id == frame_id
       end
 
       #
@@ -79,7 +79,7 @@ module Ferrum
       # @return [Boolean]
       #
       def finished?
-        blocked? || !response.nil? || !error.nil?
+        blocked? || response&.loaded? || !error.nil? || ping?
       end
 
       #
@@ -100,6 +100,33 @@ module Ferrum
         !intercepted_request.nil?
       end
 
+      #
+      # Determines if the exchange is XHR.
+      #
+      # @return [Boolean]
+      #
+      def xhr?
+        !!request&.xhr?
+      end
+
+      #
+      # Determines if the exchange is a redirect.
+      #
+      # @return [Boolean]
+      #
+      def redirect?
+        response&.redirect?
+      end
+
+      #
+      # Determines if the exchange is ping.
+      #
+      # @return [Boolean]
+      #
+      def ping?
+        !!request&.ping?
+      end
+
       #
       # Returns request's URL.
       #