bundler 4.0.21 → 4.1.0.beta1

data/lib/bundler/fetcher/compact_index.rb

@@ -2,6 +2,15 @@
 
 require_relative "base"
 require_relative "../worker"
+require "rubygems/compact_index_client"
+
+# Route the client's filesystem access through Bundler's helper so
+# permission problems raise friendly Bundler errors. The respond_to?
+# check covers a host RubyGems that already loaded its own copy of the
+# client from before the hook existed.
+if Gem::CompactIndexClient.respond_to?(:filesystem_access=)
+  Gem::CompactIndexClient.filesystem_access = Bundler::SharedHelpers.method(:filesystem_access)
+end
 
 module Bundler
   class Fetcher
@@ -11,12 +20,12 @@ module Bundler
         undef_method(method_name)
         define_method(method_name) do |*args, &blk|
           method.bind_call(self, *args, &blk)
-        rescue NetworkDownError, CompactIndexClient::Updater::MismatchedChecksumError => e
+        rescue NetworkDownError, Gem::CompactIndexClient::Updater::MismatchedChecksumError => e
           raise HTTPError, e.message
         rescue AuthenticationRequiredError, BadAuthenticationError
           # Fail since we got a 401 from the server.
           raise
-        rescue HTTPError => e
+        rescue HTTPError, Gem::CompactIndexClient::Error => e
           Bundler.ui.trace(e)
           nil
         end
@@ -35,7 +44,7 @@ module Bundler
         until remaining_gems.empty?
           log_specs { "Looking up gems #{remaining_gems.inspect}" }
           deps = fetch_gem_infos(remaining_gems).flatten(1)
-          next_gems = deps.flat_map {|d| d[CompactIndexClient::INFO_DEPS].flat_map(&:first) }.uniq
+          next_gems = deps.flat_map {|d| d[Gem::CompactIndexClient::INFO_DEPS].flat_map(&:first) }.uniq
           deps.each {|dep| gem_info << dep }
           complete_gems.concat(deps.map(&:first)).uniq!
           remaining_gems = next_gems - complete_gems
@@ -53,7 +62,7 @@ module Bundler
         end
         # Read info file checksums out of /versions, so we can know if gems are up to date
         compact_index_client.available?
-      rescue CompactIndexClient::Updater::MismatchedChecksumError => e
+      rescue Gem::CompactIndexClient::Updater::MismatchedChecksumError => e
         Bundler.ui.debug(e.message)
         nil
       end
@@ -75,7 +84,7 @@ module Bundler
       def compact_index_client
         @compact_index_client ||=
           SharedHelpers.filesystem_access(cache_path) do
-            CompactIndexClient.new(cache_path, client_fetcher)
+            Gem::CompactIndexClient.new(cache_path, client_fetcher)
           end
       end
 

data/lib/bundler/fetcher/connection_pools.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "rubygems/request"
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    # Hands out pools of configured Gem::Net::HTTP connections, compatible
+    # with the checkout/checkin interface that Gem::Request expects.
+    # Connection policy (proxy, SSL, timeouts) follows Bundler settings
+    # first, falling back to RubyGems configuration.
+    class ConnectionPools
+      # A fixed-size pool of connections to a single host, sharing the
+      # interface of Gem::Request::HTTPPool.
+      class Pool
+        attr_reader :cert_files, :proxy_uri
+
+        def initialize(connections, uri, proxy_uri, size)
+          @connections = connections
+          @uri = uri
+          @proxy_uri = proxy_uri
+          @cert_files = connections.cert_files
+          @queue = Thread::SizedQueue.new(size)
+          size.times { @queue.push(nil) }
+        end
+
+        def checkout
+          @queue.pop || @connections.build_connection(@uri, @proxy_uri)
+        end
+
+        def checkin(connection)
+          @queue.push(connection)
+        end
+      end
+
+      attr_reader :override_headers, :cert_files
+
+      def initialize(size:, timeout:)
+        @size = size
+        @timeout = timeout
+        @override_headers = {}
+        @cert_files = Gem::Request.get_cert_files
+        @pools = {}
+        @pool_mutex = Thread::Mutex.new
+      end
+
+      # Performs a GET request through Gem::Request, which handles resetting
+      # and retrying stale connections, and returns the raw response.
+      def request(uri, headers = nil)
+        Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, pool_for(uri)).fetch do |request|
+          @override_headers.each {|key, value| request[key] = value }
+          headers&.each {|key, value| request[key] = value }
+        end
+      end
+
+      def pool_for(uri)
+        key = [uri.scheme, uri.hostname, uri.port]
+        @pool_mutex.synchronize do
+          @pools[key] ||= Pool.new(self, uri, proxy_for(uri), @size)
+        end
+      end
+
+      # The proxy that will be used for +uri+, or nil. RubyGems configuration
+      # takes precedence over the environment, like Gem::RemoteFetcher.
+      # Unlike Gem::Request, an https URI falls back to `http_proxy` when no
+      # https-specific proxy is set, which is what the previous
+      # net-http-persistent based transport did.
+      def proxy_for(uri)
+        proxy = if config_proxy = Gem.configuration[:http_proxy]
+          Gem::Request.proxy_uri(config_proxy)
+        else
+          env_proxy_for(uri)
+        end
+        return unless proxy
+        # NO_PROXY="*" bypasses the proxy for every host, which
+        # Gem::URI::Generic.use_proxy? does not understand.
+        return if no_proxy_env.strip == "*"
+        return unless Gem::URI::Generic.use_proxy?(uri.hostname, nil, uri.port, no_proxy_env)
+        proxy
+      end
+
+      def build_connection(uri, proxy_uri) # :nodoc:
+        args = [uri.hostname, uri.port]
+        args += if proxy_uri
+          [proxy_uri.hostname, proxy_uri.port,
+           Gem::UriFormatter.new(proxy_uri.user).unescape,
+           Gem::UriFormatter.new(proxy_uri.password).unescape]
+        else
+          [nil, nil]
+        end
+
+        connection = Gem::Request::ConnectionPools.client.new(*args)
+        configure_ssl(connection) if uri.scheme == "https"
+        connection.open_timeout = @timeout
+        connection.read_timeout = @timeout
+        # Stale connections are already resent by Gem::Request#perform_request,
+        # and Bundler::Retry retries whole requests, so the Gem::Net::HTTP
+        # level retry would only multiply the time spent on a timing out host.
+        connection.max_retries = 0
+        connection.start
+        connection
+      end
+
+      private
+
+      def env_proxy_for(uri)
+        proxy = Gem::Request.get_proxy_from_env(uri.scheme)
+        proxy = Gem::Request.get_proxy_from_env("http") if proxy == :no_proxy && uri.scheme == "https"
+        proxy == :no_proxy ? nil : proxy
+      end
+
+      def no_proxy_env
+        ENV["no_proxy"] || ENV["NO_PROXY"] || ""
+      end
+
+      def configure_ssl(connection)
+        connection.use_ssl = true
+        connection.verify_mode = Bundler.settings[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
+        connection.cert_store = bundler_cert_store
+
+        ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
+                          (Gem.configuration.ssl_client_cert if
+                            Gem.configuration.respond_to?(:ssl_client_cert))
+        return unless ssl_client_cert
+
+        pem = File.read(ssl_client_cert)
+        connection.cert = OpenSSL::X509::Certificate.new(pem)
+        connection.key  = OpenSSL::PKey.read(pem)
+      end
+
+      def bundler_cert_store
+        store = OpenSSL::X509::Store.new
+        ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
+                      (Gem.configuration.ssl_ca_cert if
+                        Gem.configuration.respond_to?(:ssl_ca_cert))
+        if ssl_ca_cert
+          if File.directory? ssl_ca_cert
+            store.add_path ssl_ca_cert
+          else
+            store.add_file ssl_ca_cert
+          end
+        else
+          store.set_default_paths
+          cert_files.each {|c| store.add_file c }
+        end
+        store
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/downloader.rb

@@ -6,14 +6,19 @@ module Bundler
       HTTP_NON_RETRYABLE_ERRORS = [
         SocketError,
         Errno::EADDRNOTAVAIL,
+        Errno::ECONNREFUSED,
+        Errno::EHOSTDOWN,
+        Errno::EHOSTUNREACH,
         Errno::ENETDOWN,
         Errno::ENETUNREACH,
-        Gem::Net::HTTP::Persistent::Error,
-        Errno::EHOSTUNREACH,
       ].freeze
 
+      # The vendored net-http raises Gem::Timeout::Error, but when Gem::Net is
+      # the real Net (hosts without a vendored net-http), timeouts are plain
+      # Timeout::Error subclasses instead.
       HTTP_RETRYABLE_ERRORS = [
         Gem::Timeout::Error,
+        *(::Timeout::Error if defined?(::Timeout::Error)),
         EOFError,
         Errno::EINVAL,
         Errno::ECONNRESET,
@@ -25,11 +30,11 @@ module Bundler
         Zlib::BufError,
       ].freeze
 
-      attr_reader :connection
+      attr_reader :connections
       attr_reader :redirect_limit
 
-      def initialize(connection, redirect_limit)
-        @connection = connection
+      def initialize(connections, redirect_limit)
+        @connections = connections
         @redirect_limit = redirect_limit
       end
 
@@ -84,23 +89,25 @@ module Bundler
         filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
 
         Bundler.ui.debug "HTTP GET #{filtered_uri}"
-        req = Gem::Net::HTTP::Get.new uri.request_uri, headers
-        if uri.user
-          user = CGI.unescape(uri.user)
-          password = uri.password ? CGI.unescape(uri.password) : nil
-          req.basic_auth(user, password)
+        connections.request(uri, headers)
+      rescue Gem::RemoteFetcher::FetchError => e
+        Bundler.ui.trace e
+
+        case e.message
+        when /certificate verify failed/
+          raise CertificateFailureError.new(uri)
+        when /host is down|host down/i
+          raise network_down_error(uri, filtered_uri)
+        else
+          raise HTTPError, "Network error while fetching #{filtered_uri}" \
+              " (#{e})"
         end
-        connection.request(uri, req)
       rescue OpenSSL::SSL::SSLError
         raise CertificateFailureError.new(uri)
       rescue *HTTP_NON_RETRYABLE_ERRORS => e
         Bundler.ui.trace e
 
-        host = uri.host
-        host_port = "#{host}:#{uri.port}"
-        host = host_port if filtered_uri.to_s.include?(host_port)
-        raise NetworkDownError, "Could not reach host #{host}. Check your network " \
-          "connection and try again."
+        raise network_down_error(uri, filtered_uri)
       rescue *HTTP_RETRYABLE_ERRORS => e
         Bundler.ui.trace e
 
@@ -110,6 +117,14 @@ module Bundler
 
       private
 
+      def network_down_error(uri, filtered_uri)
+        host = uri.host
+        host_port = "#{host}:#{uri.port}"
+        host = host_port if filtered_uri.to_s.include?(host_port)
+        NetworkDownError.new("Could not reach host #{host}. Check your network " \
+          "connection and try again.")
+      end
+
       def https?(uri)
         uri.scheme&.casecmp("https")&.zero?
       end

data/lib/bundler/fetcher.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "vendored_persistent"
+require_relative "vendored_net_http"
 require_relative "vendored_timeout"
 require_relative "vendored_securerandom"
 require "zlib"
@@ -10,6 +10,7 @@ module Bundler
   class Fetcher
     autoload :Base, File.expand_path("fetcher/base", __dir__)
     autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__)
+    autoload :ConnectionPools, File.expand_path("fetcher/connection_pools", __dir__)
     autoload :Downloader, File.expand_path("fetcher/downloader", __dir__)
     autoload :Dependency, File.expand_path("fetcher/dependency", __dir__)
     autoload :Index, File.expand_path("fetcher/index", __dir__)
@@ -137,7 +138,7 @@ module Bundler
       @remote = remote
 
       Socket.do_not_reverse_lookup = true
-      connection # create persistent connection
+      connection # set up the connection pools eagerly so SSL support is checked upfront
     end
 
     def uri
@@ -176,13 +177,13 @@ module Bundler
     def specs(gem_names, source)
       index = Bundler::Index.new
 
-      fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata|
+      fetch_specs(gem_names).each do |name, version, suffix, dependencies, metadata|
         spec = if dependencies
-          EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es|
+          EndpointSpecification.new(name, version, suffix, self, dependencies, metadata).tap do |es|
             source.checksum_store.replace(es, es.checksum)
           end
         else
-          RemoteSpecification.new(name, version, platform, self)
+          RemoteSpecification.new(name, version, suffix, self)
         end
         spec.source = source
         spec.remote = @remote
@@ -237,7 +238,7 @@ module Bundler
     end
 
     def http_proxy
-      return unless uri = connection.proxy_uri
+      return unless uri = connection.proxy_for(remote_uri)
       uri.to_s
     end
 
@@ -307,28 +308,7 @@ module Bundler
           end
         end
 
-        con = Gem::Net::HTTP::Persistent.new name: "bundler", proxy: :ENV
-        if gem_proxy = Gem.configuration[:http_proxy]
-          con.proxy = Gem::URI.parse(gem_proxy) if gem_proxy != :no_proxy
-        end
-
-        if remote_uri.scheme == "https"
-          con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
-            OpenSSL::SSL::VERIFY_PEER)
-          con.cert_store = bundler_cert_store
-        end
-
-        ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
-                          (Gem.configuration.ssl_client_cert if
-                            Gem.configuration.respond_to?(:ssl_client_cert))
-        if ssl_client_cert
-          pem = File.read(ssl_client_cert)
-          con.cert = OpenSSL::X509::Certificate.new(pem)
-          con.key  = OpenSSL::PKey.read(pem)
-        end
-
-        con.read_timeout = Fetcher.api_timeout
-        con.open_timeout = Fetcher.api_timeout
+        con = ConnectionPools.new(size: Bundler.settings.processor_count, timeout: Fetcher.api_timeout)
         con.override_headers["User-Agent"] = user_agent
         con.override_headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
         con
@@ -341,25 +321,6 @@ module Bundler
       paths.find {|path| File.file? path }
     end
 
-    def bundler_cert_store
-      store = OpenSSL::X509::Store.new
-      ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
-                    (Gem.configuration.ssl_ca_cert if
-                      Gem.configuration.respond_to?(:ssl_ca_cert))
-      if ssl_ca_cert
-        if File.directory? ssl_ca_cert
-          store.add_path ssl_ca_cert
-        else
-          store.add_file ssl_ca_cert
-        end
-      else
-        store.set_default_paths
-        require "rubygems/request"
-        Gem::Request.get_cert_files.each {|c| store.add_file c }
-      end
-      store
-    end
-
     def remote_uri
       @remote.uri
     end

data/lib/bundler/injector.rb

@@ -89,10 +89,10 @@ module Bundler
     def version_prefix
       if @options[:strict]
         "= "
-      elsif @options[:optimistic]
-        ">= "
-      else
+      elsif @options[:pessimistic]
         "~> "
+      else
+        ">= "
       end
     end
 

data/lib/bundler/inline.rb

@@ -62,13 +62,52 @@ def gemfile(force_latest_compatible = false, options = {}, &gemfile)
       definition.validate_runtime!
 
       if force_latest_compatible || definition.missing_specs?
-        Bundler.settings.temporary(inline: true, no_install: false) do
-          installer = Bundler::Installer.install(Bundler.root, definition, system: true)
-          installer.post_install_messages.each do |name, message|
-            Bundler.ui.info "Post-install message from #{name}:\n#{message}"
+        do_install = -> do
+          Bundler.settings.temporary(inline: true, no_install: false) do
+            installer = Bundler::Installer.install(Bundler.root, definition, system: true)
+            installer.post_install_messages.each do |name, message|
+              Bundler.ui.info "Post-install message from #{name}:\n#{message}"
+            end
           end
         end
+
+        # When possible we do the install in a subprocess because to install
+        # gems we need to require some default gems like `securerandom` etc
+        # which may later conflict with the Gemfile requirements.
+        installed_in_fork = false
+        if Process.respond_to?(:fork)
+          Gem.load_yaml # Avoid NameError on Ruby 3.2's safe_yaml.rb after Gem::Specification.reset
+          _, status = Process.waitpid2(Process.fork do
+                                         $VERBOSE = nil
+                                         do_install.call end)
+          exit(status.exitstatus || status.to_i) unless status.success?
+
+          installed_in_fork = true
+
+          Bundler.reset!
+          Gem::Specification.reset
+          Bundler.instance_variable_set(:@bundle_path, Pathname.new(Gem.dir))
+
+          builder = Bundler::Dsl.new
+          builder.instance_eval(&gemfile)
+          builder.check_primary_source_safety
+
+          definition = builder.to_definition(nil, true)
+          definition.validate_runtime!
+        else
+          do_install.call
+        end
+      end
+
+      configure_forked_definition = ->(d) do
+        d.sources.rubygems_sources.each(&:remote!)
+        d.sources.git_sources.each do |source|
+          source.cached!
+          source.instance_variable_set(:@copied, true)
+        end
+        def d.lock(*); end
       end
+      configure_forked_definition.call(definition) if installed_in_fork
 
       begin
         runtime = Bundler::Runtime.new(nil, definition).setup
@@ -84,6 +123,7 @@ def gemfile(force_latest_compatible = false, options = {}, &gemfile)
         builder.dependencies.delete_if {|d| d.name == name }
         builder.instance_eval { gem name, activated_version }
         definition = builder.to_definition(nil, true)
+        configure_forked_definition.call(definition) if installed_in_fork
 
         retry
       end

data/lib/bundler/installer/parallel_installer.rb

@@ -20,10 +20,6 @@ module Bundler
         state == :installed
       end
 
-      def enqueued?
-        state == :enqueued
-      end
-
       def enqueue_with_priority?
         state == :installable && spec.extensions.any?
       end
@@ -100,7 +96,8 @@ module Bundler
       handle_error if failed_specs.any?
       @specs
     ensure
-      worker_pool&.stop
+      @worker_pool&.stop
+      @download_worker_pool&.stop
     end
 
     private
@@ -170,17 +167,18 @@ module Bundler
       end
     end
 
+    def download_worker_pool
+      @download_worker_pool ||= Bundler::Worker.new(@size, "Gem Downloader",
+        ->(spec_install, worker_num) { do_download(spec_install, worker_num) }, response_queue: response_queue)
+    end
+
     def worker_pool
-      @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num|
-        case spec_install.state
-        when :enqueued
-          do_download(spec_install, worker_num)
-        when :installable
-          do_install(spec_install, worker_num)
-        else
-          spec_install
-        end
-      }
+      @worker_pool ||= Bundler::Worker.new(@size, "Parallel Installer",
+        ->(spec_install, worker_num) { do_install(spec_install, worker_num) }, response_queue: response_queue)
+    end
+
+    def response_queue
+      @response_queue ||= Thread::Queue.new
     end
 
     def do_download(spec_install, worker_num)
@@ -218,24 +216,24 @@ module Bundler
       spec_install
     end
 
-    # Dequeue a spec and save its post-install message and then enqueue the
-    # remaining specs.
-    # Some specs might've had to wait til this spec was installed to be
-    # processed so the call to `enqueue_specs` is important after every
-    # dequeue.
+    # Process one completed download or installation. Downloads can finish
+    # before their dependencies are installed, so check all downloaded specs
+    # after each completion and enqueue any that are now installable.
     def process_specs(installed_specs)
       spec = worker_pool.deq
 
       if spec.installed?
         installed_specs[spec.name] = true
-        return
       elsif spec.failed?
         return
-      elsif spec.ready_to_install?(installed_specs)
-        spec.state = :installable
       end
 
-      worker_pool.enq(spec, priority: spec.enqueue_with_priority?)
+      @specs.each do |candidate|
+        next unless candidate.ready_to_install?(installed_specs)
+
+        candidate.state = :installable
+        worker_pool.enq(candidate, priority: candidate.enqueue_with_priority?)
+      end
     end
 
     def finished_installing?
@@ -274,11 +272,8 @@ module Bundler
       t
     end
 
-    # Keys in the remains hash represent uninstalled gems specs.
-    # We enqueue all gem specs that do not have any dependencies.
-    # Later we call this lambda again to install specs that depended on
-    # previously installed specifications. We continue until all specs
-    # are installed.
+    # Queue every missing spec for download. `process_specs` schedules each
+    # downloaded spec for installation once its dependencies are installed.
     def enqueue_specs(installed_specs)
       @specs.each do |spec|
         if spec.installed?
@@ -287,7 +282,7 @@ module Bundler
         end
 
         spec.state = :enqueued
-        worker_pool.enq spec
+        download_worker_pool.enq spec
       end
     end
   end

data/lib/bundler/installer.rb

@@ -228,18 +228,32 @@ module Bundler
     end
 
     def ensure_specs_are_compatible!
+      overrides = @definition.overrides
       @definition.specs.each do |spec|
-        unless spec.matches_current_ruby?
-          raise InstallError, "#{spec.full_name} requires ruby version #{spec.required_ruby_version}, " \
-            "which is incompatible with the current version, #{Gem.ruby_version}"
+        unless spec.matches_current_ruby_with_overrides?(overrides)
+          raise InstallError, incompatibility_message(spec, "requires ruby version #{spec.required_ruby_version}, " \
+            "which is incompatible with the current version, #{Gem.ruby_version}")
         end
-        unless spec.matches_current_rubygems?
-          raise InstallError, "#{spec.full_name} requires rubygems version #{spec.required_rubygems_version}, " \
-            "which is incompatible with the current version, #{Gem.rubygems_version}"
+        unless spec.matches_current_rubygems_with_overrides?(overrides)
+          raise InstallError, incompatibility_message(spec, "requires rubygems version #{spec.required_rubygems_version}, " \
+            "which is incompatible with the current version, #{Gem.rubygems_version}")
         end
       end
     end
 
+    # Build the error raised when a locked spec can't run on the current Ruby or
+    # RubyGems. When the offending spec is a platform-specific variant locked in
+    # frozen mode, point users at the fix: re-resolving outside frozen mode adds
+    # the Ruby (non platform-specific) fallback variant to the lockfile.
+    def incompatibility_message(spec, reason)
+      message = "#{spec.full_name} #{reason}"
+      return message unless Bundler.frozen_bundle? && spec.platform != Gem::Platform::RUBY
+
+      message + "\n\nThe lockfile only includes the #{spec.platform} variant of #{spec.name}. " \
+        "If a compatible Ruby variant is available, run `bundle install` outside of frozen/deployment mode " \
+        "to add it to the lockfile as a fallback, then commit the updated #{SharedHelpers.relative_lockfile_path}."
+    end
+
     def lock
       @definition.lock
     end