ferrum 0.18.0 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7bb51607dd8307f471fa1fcf5214a392485caf1620588798c60dadd6fb41382
-  data.tar.gz: d4ce789910ecec970a5a33ca3843ca79b148934b75397ec19df58dd81cca49af
+  metadata.gz: 4a719b34dd1893e7c0ba2b71e121e1a2b7cc7950b96a1e462ac6a3a9f36c04e0
+  data.tar.gz: 4c96a549ff3aa00d76420b43c2f7b4870cacb712d390efe7394ad2a8245f6825
 SHA512:
-  metadata.gz: b77f1fb177afcd34a22b4c82f1cb31469cf319edfaf6d67758091cd87d10a2227fdeaa51875979d5cc6e9e357bc0df0ab3ab182920525abcf5f3d81133e34c86
-  data.tar.gz: 40d23db769aba58e34709d187ec680b10ce630826d6174ddf829fa15d2e7095d99eea0fc7729c75640505a8290ad6caf221c8837d50df2dd4c0671b015cffd75
+  metadata.gz: 1a4e9981f2e6334b2862a073eb645b893bbebd42399f0e4b3090d98ea3d69ab0d3104bebe34f86e66b3ec2ea01b827efa28b75c2be55482de6f89a8b082b2113
+  data.tar.gz: acab89439a3655e51c3173bc30199bbccb6511567803fa16f066b26fbd6e5934f75492b6ed6383c8c59a27e397e23c781226abd439c551bc6be63c001868bc3a

data/README.md

@@ -3,7 +3,7 @@
 ## [Documentation](https://docs.rubycdp.com/docs/ferrum/introduction)
 
 <img align="right"
-     width="320" height="241"
+     width="160" height="160"
      alt="Ferrum logo"
      src="https://raw.githubusercontent.com/rubycdp/ferrum/main/logo.svg?sanitize=true">
 
@@ -18,6 +18,8 @@ browsers.
 * [Cuprite](https://github.com/rubycdp/cuprite) is a pure Ruby driver for [Capybara](https://github.com/teamcapybara/capybara) based on Ferrum.
 * [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework based on Ferrum and Mechanize.
 
+<br clear="right">
+
 ## Install
 
 There's no official Chrome or Chromium package for Linux don't install it this

data/lib/ferrum/browser/binary.rb

@@ -68,7 +68,7 @@ module Ferrum
         [paths, exts]
       end
 
-      # rubocop:disable Style/CollectionCompact
+      # rubocop:disable-next Style/CollectionCompact
       def lazy_find(cmds)
         cmds.lazy.map do |cmd, path, ext|
           absolute_path = File.absolute_path(cmd)
@@ -81,7 +81,6 @@ module Ferrum
           cmd
         end.reject(&:nil?) # .compact isn't defined on Enumerator::Lazy
       end
-      # rubocop:enable Style/CollectionCompact
     end
   end
 end

data/lib/ferrum/browser/command.rb

@@ -17,8 +17,6 @@ module Ferrum
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
       def self.build(options, user_data_dir)
         defaults = case options.browser_name
-                   when :firefox
-                     Options::Firefox.options
                    when :chrome, :opera, :edge, nil
                      Options::Chrome.options
                    else

data/lib/ferrum/browser/options/base.rb

@@ -8,10 +8,9 @@ module Ferrum
     class Options
       #
       # Abstract base for browser-specific default option builders. Subclasses
-      # (e.g. {Options::Chrome}, {Options::Firefox}) declare their own
-      # `DEFAULT_OPTIONS` and `PLATFORM_PATH` constants and implement
-      # {#merge_required}/{#merge_default} to produce the final CLI flags used
-      # by {Command}.
+      # (e.g. {Options::Chrome}) declare their own `DEFAULT_OPTIONS` and
+      # `PLATFORM_PATH` constants and implement {#merge_required}/{#merge_default}
+      # to produce the final CLI flags used by {Command}.
       #
       class Base
         include Singleton
@@ -71,7 +70,7 @@ module Ferrum
         #
         # Merges the CLI flags that are always required to drive the browser
         # over CDP (e.g. remote debugging port, user data dir) into `flags`.
-        # Abstract; overridden by {Options::Chrome}/{Options::Firefox}.
+        # Abstract; overridden by {Options::Chrome}.
         #
         # @param [Hash{String => Object}] flags
         #   The flags accumulated so far.
@@ -92,7 +91,7 @@ module Ferrum
         #
         # Merges this browser's default CLI flags (see {#to_h}) into `flags`,
         # unless `options.ignore_default_browser_options` is set. Abstract;
-        # overridden by {Options::Chrome}/{Options::Firefox}.
+        # overridden by {Options::Chrome}.
         #
         # @param [Hash{String => Object}] flags
         #   The flags accumulated so far.

data/lib/ferrum/browser/options/chrome.rb

@@ -61,7 +61,6 @@ module Ferrum
           "metrics-recording-only" => nil,
           "mute-audio" => nil,
           "no-crash-upload" => nil,
-          "no-crashpad" => nil,
           "no-default-browser-check" => nil,
           "no-first-run" => nil,
           "no-startup-window" => nil,
@@ -104,7 +103,6 @@ module Ferrum
         #
         def merge_required(flags, options, user_data_dir)
           flags = flags.merge("remote-debugging-port" => options.port,
-                              "remote-debugging-address" => options.host,
                               "window-size" => options.window_size&.join(","),
                               "user-data-dir" => user_data_dir)
 

data/lib/ferrum/browser/process/killer.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "ferrum/utils/elapsed_time"
+require "ferrum/utils/platform"
+
+module Ferrum
+  class Browser
+    class Process
+      #
+      # OS-level process termination and user-data-directory removal, kept
+      # separate from the rest of {Process} (spawning, CDP handshake, ...).
+      # Both the GC finalizer and the explicit {Process#stop} path need to
+      # run the exact same kill/cleanup logic on plain pid/path values --
+      # never on `self`. A finalizer proc that closes over the object it's
+      # attached to creates a reference cycle GC can never collect, so the
+      # finalizer would never run; keeping this logic as free functions
+      # taking plain arguments sidesteps that entirely.
+      #
+      module Killer
+        module_function
+
+        KILL_TIMEOUT = 2
+        WAIT_KILLED = 0.05
+        REAP_TIMEOUT = 0.5
+        REMOVE_DIR_RETRIES = 5
+        REMOVE_DIR_RETRY_DELAY = 0.1
+
+        #
+        # Kills the process at the given pid (and its process group, if
+        # any), escalating from TERM to KILL if it doesn't exit within
+        # {KILL_TIMEOUT} seconds.
+        #
+        # @param [Integer] pid
+        #
+        # @return [void]
+        #
+        def kill(pid)
+          if Utils::Platform.windows?
+            # Process.kill is unreliable on Windows
+            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
+            return
+          end
+
+          send_signal(pid, "TERM")
+          start = Utils::ElapsedTime.monotonic_time
+          leader_exited = false
+          loop do
+            # The leader (the pid we actually spawned and can #wait on) may
+            # exit well before the rest of its process group does -- e.g. a
+            # renderer/zygote that ignores TERM. Reaping the leader must not
+            # by itself end the loop, or that child is orphaned forever
+            # since KILL is only ever sent from the timeout branch below.
+            leader_exited ||= !::Process.wait(pid, ::Process::WNOHANG).nil?
+            break if leader_exited && !process_group_alive?(pid)
+
+            sleep(WAIT_KILLED)
+            next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
+
+            send_signal(pid, "KILL")
+            wait_reaped(pid) unless leader_exited
+            break
+          end
+        rescue Errno::ESRCH, Errno::ECHILD
+          # nop
+        end
+
+        #
+        # Waits for `pid` to become reapable, giving up after `timeout`.
+        #
+        # Deliberately a bounded poll rather than a blocking `Process.wait`.
+        # {#finalizer} installs {#kill} as an `ObjectSpace` finalizer, so
+        # it can run while the interpreter is shutting down, and a blocking
+        # wait there never returns.
+        #
+        # @param [Integer] pid
+        # @param [Float] timeout
+        #
+        # @return [Integer, nil]
+        #   The reaped pid, or `nil` if it didn't exit in time or wasn't ours.
+        #
+        def wait_reaped(pid, timeout: REAP_TIMEOUT)
+          start = Utils::ElapsedTime.monotonic_time
+          loop do
+            reaped = ::Process.wait(pid, ::Process::WNOHANG)
+            return reaped if reaped
+            return nil if Utils::ElapsedTime.timeout?(start, timeout)
+
+            sleep(WAIT_KILLED)
+          end
+        rescue Errno::ECHILD, Errno::ESRCH
+          nil
+        end
+
+        #
+        # Builds the proc registered as a browser's `ObjectSpace` finalizer,
+        # cleaning up after a browser nobody quit.
+        #
+        # Kills every pid first and removes the directory afterwards. That
+        # order is the whole point of this being one finalizer instead of
+        # two: Ruby runs an object's finalizers in registration order, and
+        # removing the user data directory while the browser is still alive
+        # means Chrome recreates it on the way out, leaking it for good.
+        #
+        # @param [Array<Integer>, Integer, nil] pids
+        #   Process ids to kill, e.g. the browser and Xvfb.
+        # @param [String, nil] path
+        #   User data directory to remove once they are dead.
+        #
+        # @return [Proc]
+        #
+        def finalizer(pids, path)
+          pids = Array(pids).compact
+          proc do
+            pids.each { |pid| kill(pid) }
+            remove_directory(path) if path
+          end
+        end
+
+        #
+        # Signals the whole process group Chrome was spawned into (it's
+        # spawned with `pgroup: true`), so its child processes (renderer,
+        # GPU, zygote, ...) are cleaned up too instead of being left
+        # orphaned. Falls back to signaling just the pid directly if
+        # there's no such process group to signal (e.g. Xvfb, which isn't
+        # spawned with `pgroup: true`) or the group can't be signaled.
+        #
+        # @param [Integer] pid
+        # @param [String] name
+        #
+        # @return [void]
+        #
+        def send_signal(pid, name)
+          ::Process.kill(name, -pid)
+        rescue Errno::EPERM, Errno::ESRCH
+          ::Process.kill(name, pid)
+        end
+
+        #
+        # Checks whether any process in pid's process group is still alive.
+        #
+        # @param [Integer] pid
+        #
+        # @return [Boolean]
+        #
+        def process_group_alive?(pid)
+          ::Process.kill(0, -pid)
+          true
+        rescue Errno::ESRCH
+          false
+        rescue Errno::EPERM
+          true
+        end
+
+        #
+        # Removes the given directory, retrying with exponential backoff on
+        # transient errors. Chrome can briefly hold file locks right after
+        # being killed, so the directory may not be removable on the first
+        # try; retrying avoids leaking temp directories in that case.
+        #
+        # @param [String] path
+        #   Directory to remove.
+        # @param [Integer] retries
+        #   Maximum number of removal attempts.
+        # @param [Float] delay
+        #   Base delay, in seconds, before the first retry; doubles on each
+        #   subsequent attempt.
+        #
+        # @return [void]
+        #
+        def remove_directory(path, retries: REMOVE_DIR_RETRIES, delay: REMOVE_DIR_RETRY_DELAY)
+          retries.times do |attempt|
+            FileUtils.remove_entry(path)
+            break
+          rescue Errno::ENOENT
+            break
+          rescue Errno::ENOTEMPTY, Errno::EBUSY, Errno::EACCES, Errno::EPERM => e
+            raise e if attempt == retries - 1
+
+            sleep(delay * (2**attempt))
+          end
+        rescue StandardError => e
+          warn("[Ferrum] Failed to remove user data dir #{path}: #{e.class}: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/process.rb

@@ -7,10 +7,11 @@ require "tmpdir"
 require "forwardable"
 require "ferrum/browser/options/base"
 require "ferrum/browser/options/chrome"
-require "ferrum/browser/options/firefox"
+require "ferrum/browser/process/killer"
 require "ferrum/browser/command"
 require "ferrum/utils/elapsed_time"
 require "ferrum/utils/platform"
+require "ferrum/utils/thread"
 
 module Ferrum
   class Browser
@@ -21,11 +22,12 @@ module Ferrum
     # stopping/restarting the process and cleaning up its user data directory.
     #
     class Process
-      KILL_TIMEOUT = 2
-      WAIT_KILLED = 0.05
-
       extend Forwardable
 
+      HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
+      WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
+      LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
+
       delegate path: :command
 
       #
@@ -40,66 +42,23 @@ module Ferrum
         new(*args).tap(&:start)
       end
 
-      #
-      # Builds a finalizer proc that kills the process with the given pid.
-      #
-      # @param [Integer] pid
-      #   Process id to kill.
-      #
-      # @return [Proc]
-      #
-      def self.process_killer(pid)
-        proc do
-          if Utils::Platform.windows?
-            # Process.kill is unreliable on Windows
-            ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
-          else
-            ::Process.kill("USR1", pid)
-            start = Utils::ElapsedTime.monotonic_time
-            while ::Process.wait(pid, ::Process::WNOHANG).nil?
-              sleep(WAIT_KILLED)
-              next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
-
-              ::Process.kill("KILL", pid)
-              ::Process.wait(pid)
-              break
-            end
-          end
-        rescue Errno::ESRCH, Errno::ECHILD
-          # nop
-        end
-      end
-
-      #
-      # Builds a finalizer proc that removes the directory at the given path.
-      #
-      # @param [String] path
-      #   Directory to remove.
-      #
-      # @return [Proc]
-      #
-      def self.directory_remover(path)
-        proc {
-          begin
-            FileUtils.remove_entry(path)
-          rescue StandardError
-            Errno::ENOENT
-          end
-        }
-      end
-
       attr_reader :host, :port, :ws_url, :pid, :command,
                   :default_user_agent, :browser_version, :protocol_version,
                   :v8_version, :webkit_version, :xvfb
 
       def initialize(options)
         @pid = @xvfb = @user_data_dir = nil
+        @requested_host = options.host
 
         if options.ws_url || options.url
           # `:ws_url` option is higher priority than `:url`, parse versions
           # and use it as a ws_url, otherwise use what has been parsed.
-          response = parse_json_version(options.ws_url || options.url)
-          self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
+          endpoint = options.ws_url || options.url
+          response = parse_json_version(endpoint)
+          ws_url = options.ws_url || reachable_ws_url(response&.[]("webSocketDebuggerUrl"), endpoint)
+          raise NoWebSocketUrlError, endpoint unless ws_url
+
+          self.ws_url = ws_url
           return
         end
 
@@ -107,10 +66,8 @@ module Ferrum
         @process_timeout = options.process_timeout
         @env = Hash(options.env)
 
-        tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
-        ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
-        @user_data_dir = tmpdir
-        @command = Command.build(options, tmpdir)
+        @user_data_dir = Dir.mktmpdir("ferrum_user_data_dir_")
+        @command = Command.build(options, @user_data_dir)
       end
 
       #
@@ -128,14 +85,11 @@ module Ferrum
           process_options[:pgroup] = true unless Utils::Platform.windows?
           process_options[:out] = process_options[:err] = write_io
 
-          if @command.xvfb?
-            @xvfb = Xvfb.start(@command.options)
-            ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
-          end
+          @xvfb = Xvfb.start(@command.options) if @command.xvfb?
 
           env = Hash(@xvfb&.to_env).merge(@env)
           @pid = ::Process.spawn(env, *@command.to_a, process_options)
-          ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
+          ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
 
           parse_ws_url(read_io, @process_timeout)
           parse_json_version(ws_url)
@@ -148,17 +102,26 @@ module Ferrum
       # Kills the browser process (and Xvfb, if running) and removes the user
       # data directory.
       #
-      # @return [void]
+      # @param [Boolean] wait
+      #   Whether to block until the process is confirmed dead and its user
+      #   data directory removed (the default), or return immediately and
+      #   run that cleanup on a background thread instead. Killing a
+      #   stubborn process group can block for up to {Killer::KILL_TIMEOUT}
+      #   seconds, plus retries removing its directory, which matters when quitting
+      #   many browsers in a hot path. `wait: false` is not used by
+      #   {#restart}, which always waits so the old process is fully gone
+      #   before the new one starts.
       #
-      def stop
-        if @pid
-          kill(@pid)
-          kill(@xvfb.pid) if @xvfb&.pid
-          @pid = nil
-        end
+      # @return [Thread, nil]
+      #   The background cleanup thread when `wait: false`; the caller can
+      #   `#join` it if they need cleanup to have finished, e.g. before
+      #   process exit or before reusing a fixed port. `nil` when `wait:
+      #   true`.
+      #
+      def stop(wait: true)
+        return sync_stop if wait
 
-        remove_user_data_dir if @user_data_dir
-        ObjectSpace.undefine_finalizer(self)
+        async_stop
       end
 
       #
@@ -189,12 +152,40 @@ module Ferrum
 
       private
 
-      def kill(pid)
-        self.class.process_killer(pid).call
+      def sync_stop
+        if @pid
+          Killer.kill(@pid)
+          Killer.kill(@xvfb.pid) if @xvfb&.pid
+          @pid = nil
+        end
+
+        remove_user_data_dir if @user_data_dir
+        ObjectSpace.undefine_finalizer(self)
+        nil
+      end
+
+      #
+      # Snapshots what needs killing/removing, clears instance state so the
+      # object looks stopped right away, and does the actual work on a
+      # background thread. The finalizer is left in place as a backup until
+      # that thread finishes, in case the process exits before it does.
+      #
+      def async_stop
+        pid = @pid
+        xvfb_pid = @xvfb&.pid
+        user_data_dir = @user_data_dir
+        @pid = @user_data_dir = nil
+
+        Utils::Thread.spawn do
+          Killer.kill(pid) if pid
+          Killer.kill(xvfb_pid) if xvfb_pid
+          Killer.remove_directory(user_data_dir) if user_data_dir
+          ObjectSpace.undefine_finalizer(self)
+        end
       end
 
       def remove_user_data_dir
-        self.class.directory_remover(@user_data_dir).call
+        Killer.remove_directory(@user_data_dir)
         @user_data_dir = nil
       end
 
@@ -210,7 +201,7 @@ module Ferrum
             read_io.wait_readable(max_time - now)
           else
             if output.match(regexp)
-              self.ws_url = output.match(regexp)[1].strip
+              self.ws_url = rewrite_host(output.match(regexp)[1].strip)
               break
             end
           end
@@ -228,6 +219,12 @@ module Ferrum
         @port = @ws_url.port
       end
 
+      def rewrite_host(url)
+        uri = Addressable::URI.parse(url)
+        uri.host = @requested_host
+        uri.to_s
+      end
+
       def close_io(*ios)
         ios.each do |io|
           io.close if io && !io.closed?
@@ -237,18 +234,7 @@ module Ferrum
       end
 
       def parse_json_version(url)
-        url = URI.join(url, "/json/version")
-
-        if %w[wss ws].include?(url.scheme)
-          url.scheme = case url.scheme
-                       when "ws"
-                         "http"
-                       when "wss"
-                         "https"
-                       end
-        end
-
-        response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
+        response = JSON.parse(::Net::HTTP.get(URI(json_version_url(url).to_s)))
 
         @v8_version = response["V8-Version"]
         @browser_version = response["Browser"]
@@ -260,6 +246,25 @@ module Ferrum
       rescue JSON::ParserError
         # nop
       end
+
+      def json_version_url(url)
+        url = Addressable::URI.parse(url).dup
+        url.scheme = HTTP_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = "/json/version"
+        url
+      end
+
+      def reachable_ws_url(ws_url, endpoint)
+        return unless ws_url
+
+        ws_url = Addressable::URI.parse(ws_url)
+        return ws_url.to_s unless LOCAL_HOSTS.include?(ws_url.host)
+
+        url = Addressable::URI.parse(endpoint).dup
+        url.scheme = WS_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = ws_url.path
+        url.to_s
+      end
     end
   end
 end

data/lib/ferrum/browser.rb

@@ -29,7 +29,7 @@ module Ferrum
     delegate %i[default_context] => :contexts
     delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
     delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
-                at_css at_xpath css xpath current_url current_title url title
+                at_css at_xpath css xpath wait_for_selector current_url current_title url title
                 body doctype content=
                 headers cookies network accessibility downloads
                 mouse keyboard
@@ -103,9 +103,8 @@ module Ferrum
     #   why you have slow responses and fix or block them rather than turn this
     #   setting off.
     #
-    # @option options [:chrome, :firefox] :browser_name (:chrome)
-    #   Sets the browser's name. **Note:** only experimental support for
-    #   `:firefox` for now.
+    # @option options [:chrome, :opera, :edge] :browser_name (:chrome)
+    #   Sets the browser's name.
     #
     # @option options [String] :browser_path
     #   Path to Chrome binary, you can also set ENV variable as
@@ -238,14 +237,23 @@ module Ferrum
     #
     # Terminates the browser process and closes the client connection.
     #
-    def quit
+    # @param [Boolean] wait
+    #   Whether to block until the process is confirmed dead and its user
+    #   data directory removed (the default), or return immediately and run
+    #   that cleanup on a background thread instead. See {Process#stop}.
+    #
+    # @return [Thread, nil]
+    #   The background cleanup thread when `wait: false`, `nil` otherwise.
+    #
+    def quit(wait: true)
       return unless @client
 
-      contexts.close_connections
+      contexts&.close_connections
 
       @client.close
-      @process.stop
+      thread = @process.stop(wait: wait)
       @client = @process = @contexts = nil
+      thread
     end
 
     #

data/lib/ferrum/client/subscriber.rb

@@ -106,7 +106,7 @@ module Ferrum
       private
 
       def start
-        @regular_thread = Utils::Thread.spawn(abort_on_exception: false) do
+        @regular_thread = Utils::Thread.spawn do
           loop do
             message = @regular.pop
             break unless message
@@ -115,7 +115,7 @@ module Ferrum
           end
         end
 
-        @priority_thread = Utils::Thread.spawn(abort_on_exception: false) do
+        @priority_thread = Utils::Thread.spawn do
           loop do
             message = @priority.pop
             break unless message
@@ -133,6 +133,11 @@ module Ferrum
         @on[event]&.each_with_index do |block, index|
           # In case of multiple callbacks we provide current index and total
           block.call(params, index, total)
+        rescue StandardError => e
+          # A raising callback used to terminate the dispatch thread, and with it every
+          # event for the rest of the browser's life: targets stopped being registered
+          # and any page created later raised `NoSuchTargetError`. Report and carry on.
+          warn("Ferrum: #{event} callback raised #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
         end
       end
     end