ferrum 0.17.2 → 0.19.0

data/lib/ferrum/browser/process/killer.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "ferrum/utils/elapsed_time"
+require "ferrum/utils/platform"
+
+module Ferrum
+  class Browser
+    class Process
+      #
+      # OS-level process termination and user-data-directory removal, kept
+      # separate from the rest of {Process} (spawning, CDP handshake, ...).
+      # Both the GC finalizer and the explicit {Process#stop} path need to
+      # run the exact same kill/cleanup logic on plain pid/path values --
+      # never on `self`. A finalizer proc that closes over the object it's
+      # attached to creates a reference cycle GC can never collect, so the
+      # finalizer would never run; keeping this logic as free functions
+      # taking plain arguments sidesteps that entirely.
+      #
+      module Killer
+        module_function
+
+        KILL_TIMEOUT = 2
+        WAIT_KILLED = 0.05
+        REAP_TIMEOUT = 0.5
+        REMOVE_DIR_RETRIES = 5
+        REMOVE_DIR_RETRY_DELAY = 0.1
+
+        #
+        # Kills the process at the given pid (and its process group, if
+        # any), escalating from TERM to KILL if it doesn't exit within
+        # {KILL_TIMEOUT} seconds.
+        #
+        # @param [Integer] pid
+        #
+        # @return [void]
+        #
+        def kill(pid)
+          if Utils::Platform.windows?
+            # Process.kill is unreliable on Windows
+            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
+            return
+          end
+
+          send_signal(pid, "TERM")
+          start = Utils::ElapsedTime.monotonic_time
+          leader_exited = false
+          loop do
+            # The leader (the pid we actually spawned and can #wait on) may
+            # exit well before the rest of its process group does -- e.g. a
+            # renderer/zygote that ignores TERM. Reaping the leader must not
+            # by itself end the loop, or that child is orphaned forever
+            # since KILL is only ever sent from the timeout branch below.
+            leader_exited ||= !::Process.wait(pid, ::Process::WNOHANG).nil?
+            break if leader_exited && !process_group_alive?(pid)
+
+            sleep(WAIT_KILLED)
+            next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
+
+            send_signal(pid, "KILL")
+            wait_reaped(pid) unless leader_exited
+            break
+          end
+        rescue Errno::ESRCH, Errno::ECHILD
+          # nop
+        end
+
+        #
+        # Waits for `pid` to become reapable, giving up after `timeout`.
+        #
+        # Deliberately a bounded poll rather than a blocking `Process.wait`.
+        # {#finalizer} installs {#kill} as an `ObjectSpace` finalizer, so
+        # it can run while the interpreter is shutting down, and a blocking
+        # wait there never returns.
+        #
+        # @param [Integer] pid
+        # @param [Float] timeout
+        #
+        # @return [Integer, nil]
+        #   The reaped pid, or `nil` if it didn't exit in time or wasn't ours.
+        #
+        def wait_reaped(pid, timeout: REAP_TIMEOUT)
+          start = Utils::ElapsedTime.monotonic_time
+          loop do
+            reaped = ::Process.wait(pid, ::Process::WNOHANG)
+            return reaped if reaped
+            return nil if Utils::ElapsedTime.timeout?(start, timeout)
+
+            sleep(WAIT_KILLED)
+          end
+        rescue Errno::ECHILD, Errno::ESRCH
+          nil
+        end
+
+        #
+        # Builds the proc registered as a browser's `ObjectSpace` finalizer,
+        # cleaning up after a browser nobody quit.
+        #
+        # Kills every pid first and removes the directory afterwards. That
+        # order is the whole point of this being one finalizer instead of
+        # two: Ruby runs an object's finalizers in registration order, and
+        # removing the user data directory while the browser is still alive
+        # means Chrome recreates it on the way out, leaking it for good.
+        #
+        # @param [Array<Integer>, Integer, nil] pids
+        #   Process ids to kill, e.g. the browser and Xvfb.
+        # @param [String, nil] path
+        #   User data directory to remove once they are dead.
+        #
+        # @return [Proc]
+        #
+        def finalizer(pids, path)
+          pids = Array(pids).compact
+          proc do
+            pids.each { |pid| kill(pid) }
+            remove_directory(path) if path
+          end
+        end
+
+        #
+        # Signals the whole process group Chrome was spawned into (it's
+        # spawned with `pgroup: true`), so its child processes (renderer,
+        # GPU, zygote, ...) are cleaned up too instead of being left
+        # orphaned. Falls back to signaling just the pid directly if
+        # there's no such process group to signal (e.g. Xvfb, which isn't
+        # spawned with `pgroup: true`) or the group can't be signaled.
+        #
+        # @param [Integer] pid
+        # @param [String] name
+        #
+        # @return [void]
+        #
+        def send_signal(pid, name)
+          ::Process.kill(name, -pid)
+        rescue Errno::EPERM, Errno::ESRCH
+          ::Process.kill(name, pid)
+        end
+
+        #
+        # Checks whether any process in pid's process group is still alive.
+        #
+        # @param [Integer] pid
+        #
+        # @return [Boolean]
+        #
+        def process_group_alive?(pid)
+          ::Process.kill(0, -pid)
+          true
+        rescue Errno::ESRCH
+          false
+        rescue Errno::EPERM
+          true
+        end
+
+        #
+        # Removes the given directory, retrying with exponential backoff on
+        # transient errors. Chrome can briefly hold file locks right after
+        # being killed, so the directory may not be removable on the first
+        # try; retrying avoids leaking temp directories in that case.
+        #
+        # @param [String] path
+        #   Directory to remove.
+        # @param [Integer] retries
+        #   Maximum number of removal attempts.
+        # @param [Float] delay
+        #   Base delay, in seconds, before the first retry; doubles on each
+        #   subsequent attempt.
+        #
+        # @return [void]
+        #
+        def remove_directory(path, retries: REMOVE_DIR_RETRIES, delay: REMOVE_DIR_RETRY_DELAY)
+          retries.times do |attempt|
+            FileUtils.remove_entry(path)
+            break
+          rescue Errno::ENOENT
+            break
+          rescue Errno::ENOTEMPTY, Errno::EBUSY, Errno::EACCES, Errno::EPERM => e
+            raise e if attempt == retries - 1
+
+            sleep(delay * (2**attempt))
+          end
+        rescue StandardError => e
+          warn("[Ferrum] Failed to remove user data dir #{path}: #{e.class}: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -7,69 +7,58 @@ require "tmpdir"
 require "forwardable"
 require "ferrum/browser/options/base"
 require "ferrum/browser/options/chrome"
-require "ferrum/browser/options/firefox"
+require "ferrum/browser/process/killer"
 require "ferrum/browser/command"
 require "ferrum/utils/elapsed_time"
 require "ferrum/utils/platform"
+require "ferrum/utils/thread"
 
 module Ferrum
   class Browser
+    #
+    # Spawns and manages the lifecycle of the browser OS process: builds the
+    # launch {Command}, starts it (optionally under {Xvfb} for headful mode),
+    # waits for the CDP WebSocket endpoint to become available, and handles
+    # stopping/restarting the process and cleaning up its user data directory.
+    #
     class Process
-      KILL_TIMEOUT = 2
-      WAIT_KILLED = 0.05
-
       extend Forwardable
 
+      HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
+      WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
+      LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
+
       delegate path: :command
 
+      #
+      # Builds and starts a new browser process.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Process]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
 
-      def self.process_killer(pid)
-        proc do
-          if Utils::Platform.windows?
-            # Process.kill is unreliable on Windows
-            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
-          else
-            ::Process.kill("USR1", pid)
-            start = Utils::ElapsedTime.monotonic_time
-            while ::Process.wait(pid, ::Process::WNOHANG).nil?
-              sleep(WAIT_KILLED)
-              next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
-
-              ::Process.kill("KILL", pid)
-              ::Process.wait(pid)
-              break
-            end
-          end
-        rescue Errno::ESRCH, Errno::ECHILD
-          # nop
-        end
-      end
-
-      def self.directory_remover(path)
-        proc {
-          begin
-            FileUtils.remove_entry(path)
-          rescue StandardError
-            Errno::ENOENT
-          end
-        }
-      end
-
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
                   :v8_version, :webkit_version, :xvfb
 
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
+        @requested_host = options.host
 
         if options.ws_url || options.url
           # `:ws_url` option is higher priority than `:url`, parse versions
           # and use it as a ws_url, otherwise use what has been parsed.
-          response = parse_json_version(options.ws_url || options.url)
-          self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
+          endpoint = options.ws_url || options.url
+          response = parse_json_version(endpoint)
+          ws_url = options.ws_url || reachable_ws_url(response&.[]("webSocketDebuggerUrl"), endpoint)
+          raise NoWebSocketUrlError, endpoint unless ws_url
+
+          self.ws_url = ws_url
           return
         end
 
@@ -77,12 +66,15 @@ module Ferrum
         @process_timeout = options.process_timeout
         @env = Hash(options.env)
 
-        tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
-        ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
-        @user_data_dir = tmpdir
-        @command = Command.build(options, tmpdir)
+        @user_data_dir = Dir.mktmpdir("ferrum_user_data_dir_")
+        @command = Command.build(options, @user_data_dir)
       end
 
+      #
+      # Spawns the browser process and waits for it to become reachable over CDP.
+      #
+      # @return [void]
+      #
       def start
         # Don't do anything as browser is already running as external process.
         return if ws_url
@@ -93,14 +85,11 @@ module Ferrum
           process_options[:pgroup] = true unless Utils::Platform.windows?
           process_options[:out] = process_options[:err] = write_io
 
-          if @command.xvfb?
-            @xvfb = Xvfb.start(@command.options)
-            ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
-          end
+          @xvfb = Xvfb.start(@command.options) if @command.xvfb?
 
           env = Hash(@xvfb&.to_env).merge(@env)
           @pid = ::Process.spawn(env, *@command.to_a, process_options)
-          ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
+          ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
 
           parse_ws_url(read_io, @process_timeout)
           parse_json_version(ws_url)
@@ -109,22 +98,47 @@ module Ferrum
         end
       end
 
-      def stop
-        if @pid
-          kill(@pid)
-          kill(@xvfb.pid) if @xvfb&.pid
-          @pid = nil
-        end
-
-        remove_user_data_dir if @user_data_dir
-        ObjectSpace.undefine_finalizer(self)
+      #
+      # Kills the browser process (and Xvfb, if running) and removes the user
+      # data directory.
+      #
+      # @param [Boolean] wait
+      #   Whether to block until the process is confirmed dead and its user
+      #   data directory removed (the default), or return immediately and
+      #   run that cleanup on a background thread instead. Killing a
+      #   stubborn process group can block for up to {Killer::KILL_TIMEOUT}
+      #   seconds, plus retries removing its directory, which matters when quitting
+      #   many browsers in a hot path. `wait: false` is not used by
+      #   {#restart}, which always waits so the old process is fully gone
+      #   before the new one starts.
+      #
+      # @return [Thread, nil]
+      #   The background cleanup thread when `wait: false`; the caller can
+      #   `#join` it if they need cleanup to have finished, e.g. before
+      #   process exit or before reusing a fixed port. `nil` when `wait:
+      #   true`.
+      #
+      def stop(wait: true)
+        return sync_stop if wait
+
+        async_stop
       end
 
+      #
+      # Stops and starts the browser process again.
+      #
+      # @return [void]
+      #
       def restart
         stop
         start
       end
 
+      #
+      # Custom inspection that omits noisy internal command details.
+      #
+      # @return [String]
+      #
       def inspect
         "#<#{self.class} " \
           "@user_data_dir=#{@user_data_dir.inspect} " \
@@ -138,12 +152,40 @@ module Ferrum
 
       private
 
-      def kill(pid)
-        self.class.process_killer(pid).call
+      def sync_stop
+        if @pid
+          Killer.kill(@pid)
+          Killer.kill(@xvfb.pid) if @xvfb&.pid
+          @pid = nil
+        end
+
+        remove_user_data_dir if @user_data_dir
+        ObjectSpace.undefine_finalizer(self)
+        nil
+      end
+
+      #
+      # Snapshots what needs killing/removing, clears instance state so the
+      # object looks stopped right away, and does the actual work on a
+      # background thread. The finalizer is left in place as a backup until
+      # that thread finishes, in case the process exits before it does.
+      #
+      def async_stop
+        pid = @pid
+        xvfb_pid = @xvfb&.pid
+        user_data_dir = @user_data_dir
+        @pid = @user_data_dir = nil
+
+        Utils::Thread.spawn do
+          Killer.kill(pid) if pid
+          Killer.kill(xvfb_pid) if xvfb_pid
+          Killer.remove_directory(user_data_dir) if user_data_dir
+          ObjectSpace.undefine_finalizer(self)
+        end
       end
 
       def remove_user_data_dir
-        self.class.directory_remover(@user_data_dir).call
+        Killer.remove_directory(@user_data_dir)
         @user_data_dir = nil
       end
 
@@ -159,7 +201,7 @@ module Ferrum
             read_io.wait_readable(max_time - now)
           else
             if output.match(regexp)
-              self.ws_url = output.match(regexp)[1].strip
+              self.ws_url = rewrite_host(output.match(regexp)[1].strip)
               break
             end
           end
@@ -177,6 +219,12 @@ module Ferrum
         @port = @ws_url.port
       end
 
+      def rewrite_host(url)
+        uri = Addressable::URI.parse(url)
+        uri.host = @requested_host
+        uri.to_s
+      end
+
       def close_io(*ios)
         ios.each do |io|
           io.close if io && !io.closed?
@@ -186,18 +234,7 @@ module Ferrum
       end
 
       def parse_json_version(url)
-        url = URI.join(url, "/json/version")
-
-        if %w[wss ws].include?(url.scheme)
-          url.scheme = case url.scheme
-                       when "ws"
-                         "http"
-                       when "wss"
-                         "https"
-                       end
-        end
-
-        response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
+        response = JSON.parse(::Net::HTTP.get(URI(json_version_url(url).to_s)))
 
         @v8_version = response["V8-Version"]
         @browser_version = response["Browser"]
@@ -209,6 +246,25 @@ module Ferrum
       rescue JSON::ParserError
         # nop
       end
+
+      def json_version_url(url)
+        url = Addressable::URI.parse(url).dup
+        url.scheme = HTTP_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = "/json/version"
+        url
+      end
+
+      def reachable_ws_url(ws_url, endpoint)
+        return unless ws_url
+
+        ws_url = Addressable::URI.parse(ws_url)
+        return ws_url.to_s unless LOCAL_HOSTS.include?(ws_url.host)
+
+        url = Addressable::URI.parse(endpoint).dup
+        url.scheme = WS_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = ws_url.path
+        url.to_s
+      end
     end
   end
 end

data/lib/ferrum/browser/xvfb.rb

@@ -2,10 +2,24 @@
 
 module Ferrum
   class Browser
+    #
+    # Manages an Xvfb virtual display server, letting a "headful" browser run
+    # in environments without a real display (e.g. CI). Spawned by {Process}
+    # when the `:xvfb` option is set, and exposes the `DISPLAY` environment
+    # variable the browser process needs to attach to it.
+    #
     class Xvfb
       NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
                   "it with your package manager"
 
+      #
+      # Builds and starts a new Xvfb instance.
+      #
+      # @param [Array] args
+      #   Arguments forwarded to {#initialize}.
+      #
+      # @return [Xvfb]
+      #
       def self.start(*args)
         new(*args).tap(&:start)
       end
@@ -20,11 +34,21 @@ module Ferrum
         @display_id = (Time.now.to_f * 1000).to_i % 100_000_000
       end
 
+      #
+      # Spawns the Xvfb process on the configured display.
+      #
+      # @return [void]
+      #
       def start
         @pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
         ::Process.detach(@pid)
       end
 
+      #
+      # Environment variables needed to point the browser at this Xvfb display.
+      #
+      # @return [Hash{String => String}]
+      #
       def to_env
         { "DISPLAY" => ":#{display_id}" }
       end

data/lib/ferrum/browser.rb

@@ -3,7 +3,7 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
-require "ferrum/proxy"
+require "ferrum/worker"
 require "ferrum/client"
 require "ferrum/contexts"
 require "ferrum/browser/xvfb"
@@ -13,15 +13,25 @@ require "ferrum/browser/binary"
 require "ferrum/browser/version_info"
 
 module Ferrum
+  #
+  # The main entry point of the library. Instantiating it spawns (or connects
+  # to) a browser process and opens a CDP connection to it, exposing a single
+  # default {Page} that most of the top-level methods (`go_to`, `at_css`,
+  # `screenshot`, etc.) are delegated to for convenience.
+  #
+  # @note A `Browser` can host multiple {Context}s (like incognito profiles)
+  #   and multiple {Page}s within each of them, reachable through {#contexts}
+  #   and {#create_page}.
+  #
   class Browser
     extend Forwardable
 
     delegate %i[default_context] => :contexts
-    delegate %i[targets create_target page pages windows] => :default_context
+    delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
-                at_css at_xpath css xpath current_url current_title url title
+                at_css at_xpath css xpath wait_for_selector current_url current_title url title
                 body doctype content=
-                headers cookies network downloads
+                headers cookies network accessibility downloads
                 mouse keyboard
                 screenshot pdf mhtml viewport_size device_pixel_ratio
                 start_screencast stop_screencast
@@ -34,7 +44,8 @@ module Ferrum
 
     attr_reader :client, :process, :contexts, :options
 
-    delegate %i[timeout timeout= base_url base_url= default_user_agent default_user_agent= extensions] => :options
+    delegate %i[timeout timeout= protocol_timeout protocol_timeout=
+                base_url base_url= default_user_agent default_user_agent= extensions] => :options
     delegate %i[command] => :client
 
     #
@@ -70,26 +81,30 @@ module Ferrum
     #   When present, debug output is written to this object.
     #
     # @option options [Integer, Float] :slowmo
-    #   Set a delay in seconds to wait before sending command.
-    #   Useful companion of headless option, so that you have time to see
+    #   Set a delay in seconds to wait before sending a command.
+    #   Useful companion of a headless option, so that you have time to see
     #   changes.
     #
     # @option options [Numeric] :timeout (5)
-    #   The number of seconds we'll wait for a response when communicating with
-    #   browser.
+    #   The number of seconds we'll wait for a response when communicating
+    #   with the browser: navigations, JS evaluation, DOM queries, dispatching input, etc.
+    #
+    # @option options [Numeric] :protocol_timeout (5)
+    #   The number of seconds we'll wait for an individual internal CDP
+    #   bookkeeping call to respond, e.g. `Target.createTarget`,
+    #   `Target.attachToTarget`. These normally resolve in milliseconds.
     #
     # @option options [Boolean] :js_errors
     #   When true, JavaScript errors get re-raised in Ruby.
     #
-    # @option options [Boolean] :pending_connection_errors (false)
+    # @option options [Boolean] :pending_connection_errors (true)
     #   When main frame is still waiting for slow responses while timeout is
     #   reached {PendingConnectionsError} is raised. It's better to figure out
     #   why you have slow responses and fix or block them rather than turn this
     #   setting off.
     #
-    # @option options [:chrome, :firefox] :browser_name (:chrome)
-    #   Sets the browser's name. **Note:** only experimental support for
-    #   `:firefox` for now.
+    # @option options [:chrome, :opera, :edge] :browser_name (:chrome)
+    #   Sets the browser's name.
     #
     # @option options [String] :browser_path
     #   Path to Chrome binary, you can also set ENV variable as
@@ -172,7 +187,7 @@ module Ferrum
     ensure
       if block_given?
         page&.close
-        context.dispose if new_context
+        context&.dispose if new_context
       end
     end
 
@@ -211,19 +226,34 @@ module Ferrum
       contexts.reset
     end
 
+    #
+    # Restarts the browser process, keeping the same options.
+    #
     def restart
       quit
       start
     end
 
-    def quit
+    #
+    # Terminates the browser process and closes the client connection.
+    #
+    # @param [Boolean] wait
+    #   Whether to block until the process is confirmed dead and its user
+    #   data directory removed (the default), or return immediately and run
+    #   that cleanup on a background thread instead. See {Process#stop}.
+    #
+    # @return [Thread, nil]
+    #   The background cleanup thread when `wait: false`, `nil` otherwise.
+    #
+    def quit(wait: true)
       return unless @client
 
-      contexts.close_connections
+      contexts&.close_connections
 
       @client.close
-      @process.stop
+      thread = @process.stop(wait: wait)
       @client = @process = @contexts = nil
+      thread
     end
 
     #