ferrum 0.14 → 0.16

data/lib/ferrum/client.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+require "forwardable"
+require "ferrum/client/subscriber"
+require "ferrum/client/web_socket"
+require "ferrum/utils/thread"
+
+module Ferrum
+  class SessionClient
+    attr_reader :client, :session_id
+
+    def self.event_name(event, session_id)
+      [event, session_id].compact.join("_")
+    end
+
+    def initialize(client, session_id)
+      @client = client
+      @session_id = session_id
+    end
+
+    def command(method, async: false, **params)
+      message = build_message(method, params)
+      @client.send_message(message, async: async)
+    end
+
+    def on(event, &block)
+      @client.on(event_name(event), &block)
+    end
+
+    def subscribed?(event)
+      @client.subscribed?(event_name(event))
+    end
+
+    def respond_to_missing?(name, include_private)
+      @client.respond_to?(name, include_private)
+    end
+
+    def method_missing(name, *args, **opts, &block)
+      @client.send(name, *args, **opts, &block)
+    end
+
+    def close
+      @client.subscriber.clear(session_id: session_id)
+    end
+
+    private
+
+    def build_message(method, params)
+      @client.build_message(method, params).merge(sessionId: session_id)
+    end
+
+    def event_name(event)
+      self.class.event_name(event, session_id)
+    end
+  end
+
+  class Client
+    extend Forwardable
+    delegate %i[timeout timeout=] => :options
+
+    attr_reader :ws_url, :options, :subscriber
+
+    def initialize(ws_url, options)
+      @command_id = 0
+      @ws_url = ws_url
+      @options = options
+      @pendings = Concurrent::Hash.new
+      @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
+      @subscriber = Subscriber.new
+
+      start
+    end
+
+    def command(method, async: false, **params)
+      message = build_message(method, params)
+      send_message(message, async: async)
+    end
+
+    def send_message(message, async:)
+      if async
+        @ws.send_message(message)
+        true
+      else
+        pending = Concurrent::IVar.new
+        @pendings[message[:id]] = pending
+        @ws.send_message(message)
+        data = pending.value!(timeout)
+        @pendings.delete(message[:id])
+
+        raise DeadBrowserError if data.nil? && @ws.messages.closed?
+        raise TimeoutError unless data
+
+        error, response = data.values_at("error", "result")
+        raise_browser_error(error) if error
+        response
+      end
+    end
+
+    def on(event, &block)
+      @subscriber.on(event, &block)
+    end
+
+    def subscribed?(event)
+      @subscriber.subscribed?(event)
+    end
+
+    def session(session_id)
+      SessionClient.new(self, session_id)
+    end
+
+    def close
+      @ws.close
+      # Give a thread some time to handle a tail of messages
+      @pendings.clear
+      @thread.kill unless @thread.join(1)
+      @subscriber.close
+    end
+
+    def inspect
+      "#<#{self.class} " \
+        "@command_id=#{@command_id.inspect} " \
+        "@pendings=#{@pendings.inspect} " \
+        "@ws=#{@ws.inspect}>"
+    end
+
+    def build_message(method, params)
+      { method: method, params: params }.merge(id: next_command_id)
+    end
+
+    private
+
+    def start
+      @thread = Utils::Thread.spawn do
+        loop do
+          message = @ws.messages.pop
+          break unless message
+
+          if message.key?("method")
+            @subscriber << message
+          else
+            @pendings[message["id"]]&.set(message)
+          end
+        end
+      end
+    end
+
+    def next_command_id
+      @command_id += 1
+    end
+
+    def raise_browser_error(error)
+      case error["message"]
+      # Node has disappeared while we were trying to get it
+      when "No node with given id found",
+           "Could not find node with given id",
+           "Inspected target navigated or closed"
+        raise NodeNotFoundError, error
+      # Context is lost, page is reloading
+      when "Cannot find context with specified id"
+        raise NoExecutionContextError, error
+      when "No target with given id found"
+        raise NoSuchPageError
+      when /Could not compute content quads/
+        raise CoordinatesNotFoundError
+      else
+        raise BrowserError, error
+      end
+    end
+  end
+end

data/lib/ferrum/context.rb

@@ -8,9 +8,9 @@ module Ferrum
 
     attr_reader :id, :targets
 
-    def initialize(browser, contexts, id)
+    def initialize(client, contexts, id)
       @id = id
-      @browser = browser
+      @client = client
       @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::MVar.new
@@ -46,33 +46,37 @@ module Ferrum
     end
 
     def create_target
-      @browser.command("Target.createTarget",
-                       browserContextId: @id,
-                       url: "about:blank")
-      target = @pendings.take(@browser.timeout)
+      @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")
+      target = @pendings.take(@client.timeout)
       raise NoSuchTargetError unless target.is_a?(Target)
 
-      @targets.put_if_absent(target.id, target)
       target
     end
 
-    def add_target(params)
-      target = Target.new(@browser, params)
-      if target.window?
-        @targets.put_if_absent(target.id, target)
-      else
-        @pendings.put(target, @browser.timeout)
-      end
+    def add_target(params:, session_id: nil)
+      new_target = Target.new(@client, session_id, params)
+      target = @targets.put_if_absent(new_target.id, new_target)
+      target ||= new_target # `put_if_absent` returns nil if added a new value or existing if there was one already
+      @pendings.put(target, @client.timeout) if @pendings.empty?
+      target
     end
 
     def update_target(target_id, params)
-      @targets[target_id].update(params)
+      @targets[target_id]&.update(params)
     end
 
     def delete_target(target_id)
       @targets.delete(target_id)
     end
 
+    def close_targets_connection
+      @targets.each_value do |target|
+        next unless target.connected?
+
+        target.page.close_connection
+      end
+    end
+
     def dispose
       @contexts.dispose(@id)
     end

data/lib/ferrum/contexts.rb

@@ -4,12 +4,15 @@ require "ferrum/context"
 
 module Ferrum
   class Contexts
+    include Enumerable
+
     attr_reader :contexts
 
-    def initialize(browser)
+    def initialize(client)
       @contexts = Concurrent::Map.new
-      @browser = browser
+      @client = client
       subscribe
+      auto_attach
       discover
     end
 
@@ -17,6 +20,16 @@ module Ferrum
       @default_context ||= create
     end
 
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      @contexts.each(&block)
+    end
+
+    def [](id)
+      @contexts[id]
+    end
+
     def find_by(target_id:)
       context = nil
       @contexts.each_value { |c| context = c if c.target?(target_id) }
@@ -24,21 +37,25 @@ module Ferrum
     end
 
     def create(**options)
-      response = @browser.command("Target.createBrowserContext", **options)
+      response = @client.command("Target.createBrowserContext", **options)
       context_id = response["browserContextId"]
-      context = Context.new(@browser, self, context_id)
+      context = Context.new(@client, self, context_id)
       @contexts[context_id] = context
       context
     end
 
     def dispose(context_id)
       context = @contexts[context_id]
-      @browser.command("Target.disposeBrowserContext",
-                       browserContextId: context.id)
+      context.close_targets_connection
+      @client.command("Target.disposeBrowserContext", browserContextId: context.id)
       @contexts.delete(context_id)
       true
     end
 
+    def close_connections
+      @contexts.each_value(&:close_targets_connection)
+    end
+
     def reset
       @default_context = nil
       @contexts.each_key { |id| dispose(id) }
@@ -51,15 +68,26 @@ module Ferrum
     private
 
     def subscribe
-      @browser.client.on("Target.targetCreated") do |params|
+      @client.on("Target.attachedToTarget") do |params|
+        info, session_id = params.values_at("targetInfo", "sessionId")
+        next unless info["type"] == "page"
+
+        context_id = info["browserContextId"]
+        @contexts[context_id]&.add_target(session_id: session_id, params: info)
+        if params["waitingForDebugger"]
+          @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
+        end
+      end
+
+      @client.on("Target.targetCreated") do |params|
         info = params["targetInfo"]
         next unless info["type"] == "page"
 
         context_id = info["browserContextId"]
-        @contexts[context_id]&.add_target(info)
+        @contexts[context_id]&.add_target(params: info)
       end
 
-      @browser.client.on("Target.targetInfoChanged") do |params|
+      @client.on("Target.targetInfoChanged") do |params|
         info = params["targetInfo"]
         next unless info["type"] == "page"
 
@@ -67,19 +95,25 @@ module Ferrum
         @contexts[context_id]&.update_target(target_id, info)
       end
 
-      @browser.client.on("Target.targetDestroyed") do |params|
+      @client.on("Target.targetDestroyed") do |params|
         context = find_by(target_id: params["targetId"])
         context&.delete_target(params["targetId"])
       end
 
-      @browser.client.on("Target.targetCrashed") do |params|
+      @client.on("Target.targetCrashed") do |params|
         context = find_by(target_id: params["targetId"])
         context&.delete_target(params["targetId"])
       end
     end
 
     def discover
-      @browser.command("Target.setDiscoverTargets", discover: true)
+      @client.command("Target.setDiscoverTargets", discover: true)
+    end
+
+    def auto_attach
+      return unless @client.options.flatten
+
+      @client.command("Target.setAutoAttach", autoAttach: true, waitForDebuggerOnStart: true, flatten: true)
     end
   end
 end

data/lib/ferrum/cookies.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "yaml"
 require "ferrum/cookies/cookie"
 
 module Ferrum
@@ -169,10 +170,36 @@ module Ferrum
       true
     end
 
+    #
+    # Stores all cookies of current page in a file.
+    #
+    # @return [Integer]
+    #
+    # @example
+    #   browser.cookies.store # => Integer
+    #
+    def store(path = "cookies.yml")
+      File.write(path, map(&:to_h).to_yaml)
+    end
+
+    #
+    # Loads all cookies from the file and sets them for current page.
+    #
+    # @return [true]
+    #
+    # @example
+    #   browser.cookies.load # => true
+    #
+    def load(path = "cookies.yml")
+      cookies = YAML.load_file(path)
+      cookies.each { |c| set(c) }
+      true
+    end
+
     private
 
     def default_domain
-      URI.parse(@page.browser.base_url).host if @page.browser.base_url
+      URI.parse(@page.base_url).host if @page.base_url
     end
   end
 end

data/lib/ferrum/downloads.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Downloads
+    VALID_BEHAVIOR = %i[deny allow allowAndName default].freeze
+
+    def initialize(page)
+      @page = page
+      @event = Utils::Event.new.tap(&:set)
+      @files = {}
+    end
+
+    def files
+      @files.values
+    end
+
+    def wait(timeout = 5)
+      @event.reset
+      yield if block_given?
+      @event.wait(timeout)
+      @event.set
+    end
+
+    def set_behavior(save_path:, behavior: :allow)
+      raise ArgumentError unless VALID_BEHAVIOR.include?(behavior.to_sym)
+      raise Error, "supply absolute path for `:save_path` option" unless Pathname.new(save_path.to_s).absolute?
+
+      @page.command("Browser.setDownloadBehavior",
+                    browserContextId: @page.context_id,
+                    downloadPath: save_path,
+                    behavior: behavior,
+                    eventsEnabled: true)
+    end
+
+    def subscribe
+      subscribe_download_will_begin
+      subscribe_download_progress
+    end
+
+    def subscribe_download_will_begin
+      @page.on("Browser.downloadWillBegin") do |params|
+        @event.reset
+        @files[params["guid"]] = params
+      end
+    end
+
+    def subscribe_download_progress
+      @page.on("Browser.downloadProgress") do |params|
+        @files[params["guid"]].merge!(params)
+
+        case params["state"]
+        when "completed", "canceled"
+          @event.set
+        else
+          @event.reset
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/errors.rb

@@ -6,7 +6,8 @@ module Ferrum
   class NoSuchTargetError   < Error; end
   class NotImplementedError < Error; end
   class BinaryNotFoundError < Error; end
-  class EmptyPathError < Error; end
+  class EmptyPathError      < Error; end
+  class ServerError         < Error; end
 
   class StatusError < Error
     def initialize(url, message = nil)
@@ -77,6 +78,13 @@ module Ferrum
     end
   end
 
+  class InvalidScreenshotFormatError < Error
+    def initialize(format)
+      valid_formats = Page::Screenshot::SUPPORTED_SCREENSHOT_FORMAT.join(" | ")
+      super("Invalid value #{format} for option `:format` (#{valid_formats})")
+    end
+  end
+
   class BrowserError < Error
     attr_reader :response
 
@@ -98,8 +106,7 @@ module Ferrum
 
   class NoExecutionContextError < BrowserError
     def initialize(response = nil)
-      response ||= { "message" => "There's no context available" }
-      super(response)
+      super(response || { "message" => "There's no context available" })
     end
   end
 

data/lib/ferrum/headers.rb

@@ -68,7 +68,7 @@ module Ferrum
 
     def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
       options = {}
-      options[:userAgent] = user_agent || @page.browser.default_user_agent
+      options[:userAgent] = user_agent || @page.default_user_agent
       options[:acceptLanguage] = accept_language if accept_language
       options[:platform] if platform
 

data/lib/ferrum/network/exchange.rb

@@ -79,7 +79,7 @@ module Ferrum
       # @return [Boolean]
       #
       def finished?
-        blocked? || response&.loaded? || !error.nil?
+        blocked? || response&.loaded? || !error.nil? || ping?
       end
 
       #
@@ -118,6 +118,15 @@ module Ferrum
         response&.redirect?
       end
 
+      #
+      # Determines if the exchange is ping.
+      #
+      # @return [Boolean]
+      #
+      def ping?
+        !!request&.ping?
+      end
+
       #
       # Returns request's URL.
       #

data/lib/ferrum/network/intercepted_request.rb

@@ -10,9 +10,9 @@ module Ferrum
 
       attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
 
-      def initialize(page, params)
+      def initialize(client, params)
         @status = nil
-        @page = page
+        @client = client
         @params = params
         @request_id = params["requestId"]
         @frame_id = params["frameId"]
@@ -43,18 +43,18 @@ module Ferrum
         options = options.merge(body: Base64.strict_encode64(options.fetch(:body, ""))) if has_body
 
         @status = :responded
-        @page.command("Fetch.fulfillRequest", **options)
+        @client.command("Fetch.fulfillRequest", async: true, **options)
       end
 
       def continue(**options)
         options = options.merge(requestId: request_id)
         @status = :continued
-        @page.command("Fetch.continueRequest", **options)
+        @client.command("Fetch.continueRequest", async: true, **options)
       end
 
       def abort
         @status = :aborted
-        @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
+        @client.command("Fetch.failRequest", async: true, requestId: request_id, errorReason: "BlockedByClient")
       end
 
       def initial_priority

data/lib/ferrum/network/request.rb

@@ -80,6 +80,15 @@ module Ferrum
         @time ||= Time.strptime(@params["wallTime"].to_s, "%s")
       end
 
+      #
+      # Determines if a request is of type ping.
+      #
+      # @return [Boolean]
+      #
+      def ping?
+        type?("ping")
+      end
+
       #
       # Converts the request to a Hash.
       #