bundler 1.17.3 → 2.6.3

data/lib/bundler/fetcher/downloader.rb

@@ -14,67 +14,74 @@ module Bundler
       def fetch(uri, headers = {}, counter = 0)
         raise HTTPError, "Too many redirects" if counter >= redirect_limit
 
+        filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
+
         response = request(uri, headers)
-        Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}")
+        Bundler.ui.debug("HTTP #{response.code} #{response.message} #{filtered_uri}")
 
         case response
-        when Net::HTTPSuccess, Net::HTTPNotModified
+        when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified
           response
-        when Net::HTTPRedirection
-          new_uri = URI.parse(response["location"])
+        when Gem::Net::HTTPRedirection
+          new_uri = Gem::URI.parse(response["location"])
           if new_uri.host == uri.host
             new_uri.user = uri.user
             new_uri.password = uri.password
           end
           fetch(new_uri, headers, counter + 1)
-        when Net::HTTPRequestedRangeNotSatisfiable
+        when Gem::Net::HTTPRequestedRangeNotSatisfiable
           new_headers = headers.dup
           new_headers.delete("Range")
           new_headers["Accept-Encoding"] = "gzip"
           fetch(uri, new_headers)
-        when Net::HTTPRequestEntityTooLarge
+        when Gem::Net::HTTPRequestEntityTooLarge
           raise FallbackError, response.body
-        when Net::HTTPUnauthorized
+        when Gem::Net::HTTPTooManyRequests
+          raise TooManyRequestsError, response.body
+        when Gem::Net::HTTPUnauthorized
+          raise BadAuthenticationError, uri.host if uri.userinfo
           raise AuthenticationRequiredError, uri.host
-        when Net::HTTPNotFound
-          raise FallbackError, "Net::HTTPNotFound"
+        when Gem::Net::HTTPForbidden
+          raise AuthenticationForbiddenError, uri.host
+        when Gem::Net::HTTPNotFound
+          raise FallbackError, "Gem::Net::HTTPNotFound: #{filtered_uri}"
         else
-          raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
+          message = "Gem::#{response.class.name.gsub(/\AGem::/, "")}"
+          message += ": #{response.body}" unless response.body.empty?
+          raise HTTPError, message
         end
       end
 
       def request(uri, headers)
         validate_uri_scheme!(uri)
 
-        Bundler.ui.debug "HTTP GET #{uri}"
-        req = Net::HTTP::Get.new uri.request_uri, headers
+        filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
+
+        Bundler.ui.debug "HTTP GET #{filtered_uri}"
+        req = Gem::Net::HTTP::Get.new uri.request_uri, headers
         if uri.user
           user = CGI.unescape(uri.user)
           password = uri.password ? CGI.unescape(uri.password) : nil
           req.basic_auth(user, password)
         end
         connection.request(uri, req)
-      rescue NoMethodError => e
-        raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet }
-        raise LoadError.new("cannot load such file -- openssl")
       rescue OpenSSL::SSL::SSLError
         raise CertificateFailureError.new(uri)
       rescue *HTTP_ERRORS => e
         Bundler.ui.trace e
-        case e.message
-        when /host down:/, /getaddrinfo: nodename nor servname provided/
+        if e.is_a?(SocketError) || e.message.to_s.include?("host down:")
           raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \
             "connection and try again."
         else
-          raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \
+          raise HTTPError, "Network error while fetching #{filtered_uri}" \
             " (#{e})"
         end
       end
 
-    private
+      private
 
       def validate_uri_scheme!(uri)
-        return if uri.scheme =~ /\Ahttps?\z/
+        return if /\Ahttps?\z/.match?(uri.scheme)
         raise InvalidOption,
           "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \
           "Did you mean `http` or `https`?"

data/lib/bundler/fetcher/gem_remote_fetcher.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    class GemRemoteFetcher < Gem::RemoteFetcher
+      def request(*args)
+        super do |req|
+          req.delete("User-Agent") if headers["User-Agent"]
+          yield req if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/index.rb

@@ -1,51 +1,24 @@
 # frozen_string_literal: true
 
-require "bundler/fetcher/base"
-require "rubygems/remote_fetcher"
+require_relative "base"
 
 module Bundler
   class Fetcher
     class Index < Base
       def specs(_gem_names)
-        Bundler.rubygems.fetch_all_remote_specs(remote)
-      rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e
+        Bundler.rubygems.fetch_all_remote_specs(remote, gem_remote_fetcher)
+      rescue Gem::RemoteFetcher::FetchError => e
         case e.message
         when /certificate verify failed/
           raise CertificateFailureError.new(display_uri)
         when /401/
-          raise AuthenticationRequiredError, remote_uri
-        when /403/
           raise BadAuthenticationError, remote_uri if remote_uri.userinfo
           raise AuthenticationRequiredError, remote_uri
+        when /403/
+          raise AuthenticationForbiddenError, remote_uri
         else
-          Bundler.ui.trace e
-          raise HTTPError, "Could not fetch specs from #{display_uri}"
-        end
-      end
-
-      def fetch_spec(spec)
-        spec -= [nil, "ruby", ""]
-        spec_file_name = "#{spec.join "-"}.gemspec"
-
-        uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
-        if uri.scheme == "file"
-          Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(uri.path))
-        elsif cached_spec_path = gemspec_cached_path(spec_file_name)
-          Bundler.load_gemspec(cached_spec_path)
-        else
-          Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
+          raise HTTPError, "Could not fetch specs from #{display_uri} due to underlying error <#{e.message}>"
         end
-      rescue MarshalError
-        raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
-          "Your network or your gem server is probably having issues right now."
-      end
-
-    private
-
-      # cached gem specification path, if one exists
-      def gemspec_cached_path(spec_file_name)
-        paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
-        paths.find {|path| File.file? path }
       end
     end
   end

data/lib/bundler/fetcher.rb

@@ -1,22 +1,27 @@
 # frozen_string_literal: true
 
-require "bundler/vendored_persistent"
+require_relative "vendored_persistent"
+require_relative "vendored_timeout"
 require "cgi"
-require "securerandom"
+require_relative "vendored_securerandom"
 require "zlib"
 
 module Bundler
   # Handles all the fetching with the rubygems server
   class Fetcher
-    autoload :CompactIndex, "bundler/fetcher/compact_index"
-    autoload :Downloader, "bundler/fetcher/downloader"
-    autoload :Dependency, "bundler/fetcher/dependency"
-    autoload :Index, "bundler/fetcher/index"
+    autoload :Base, File.expand_path("fetcher/base", __dir__)
+    autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__)
+    autoload :Downloader, File.expand_path("fetcher/downloader", __dir__)
+    autoload :Dependency, File.expand_path("fetcher/dependency", __dir__)
+    autoload :Index, File.expand_path("fetcher/index", __dir__)
 
     # This error is raised when it looks like the network is down
     class NetworkDownError < HTTPError; end
+    # This error is raised if we should rate limit our requests to the API
+    class TooManyRequestsError < HTTPError; end
     # This error is raised if the API returns a 413 (only printed in verbose)
     class FallbackError < HTTPError; end
+
     # This is the error raised if OpenSSL fails the cert verification
     class CertificateFailureError < HTTPError
       def initialize(remote_uri)
@@ -25,28 +30,30 @@ module Bundler
           " is a chance you are experiencing a man-in-the-middle attack, but" \
           " most likely your system doesn't have the CA certificates needed" \
           " for verification. For information about OpenSSL certificates, see" \
-          " http://bit.ly/ruby-ssl. To connect without using SSL, edit your Gemfile" \
-          " sources and change 'https' to 'http'."
+          " https://railsapps.github.io/openssl-certificate-verify-failed.html."
       end
     end
+
     # This is the error raised when a source is HTTPS and OpenSSL didn't load
     class SSLError < HTTPError
       def initialize(msg = nil)
-        super msg || "Could not load OpenSSL.\n" \
-            "You must recompile Ruby with OpenSSL support or change the sources in your " \
-            "Gemfile from 'https' to 'http'. Instructions for compiling with OpenSSL " \
-            "using RVM are available at rvm.io/packages/openssl."
+        super "Could not load OpenSSL.\n" \
+          "You must recompile Ruby with OpenSSL support.\n" \
+          "original error: #{msg}\n"
       end
     end
+
     # This error is raised if HTTP authentication is required, but not provided.
     class AuthenticationRequiredError < HTTPError
       def initialize(remote_uri)
         remote_uri = filter_uri(remote_uri)
         super "Authentication is required for #{remote_uri}.\n" \
           "Please supply credentials for this source. You can do this by running:\n" \
-          " bundle config #{remote_uri} username:password"
+          "`bundle config set --global #{remote_uri} username:password`\n" \
+          "or by storing the credentials in the `#{Settings.key_for(remote_uri)}` environment variable"
       end
     end
+
     # This error is raised if HTTP authentication is provided, but incorrect.
     class BadAuthenticationError < HTTPError
       def initialize(remote_uri)
@@ -56,6 +63,16 @@ module Bundler
       end
     end
 
+    # This error is raised if HTTP authentication is correct, but lacks
+    # necessary permissions.
+    class AuthenticationForbiddenError < HTTPError
+      def initialize(remote_uri)
+        remote_uri = filter_uri(remote_uri)
+        super "Access token could not be authenticated for #{remote_uri}.\n" \
+          "Make sure it's valid and has the necessary scopes configured."
+      end
+    end
+
     # Exceptions classes that should bypass retry attempts. If your password didn't work the
     # first time, it's not going to the third time.
     NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency,
@@ -65,9 +82,9 @@ module Bundler
                   :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity,
                   :HTTPUnsupportedMediaType, :HTTPVersionNotSupported].freeze
     FAIL_ERRORS = begin
-      fail_errors = [AuthenticationRequiredError, BadAuthenticationError, FallbackError]
-      fail_errors << Gem::Requirement::BadRequirementError if defined?(Gem::Requirement::BadRequirementError)
-      fail_errors.concat(NET_ERRORS.map {|e| SharedHelpers.const_get_safely(e, Net) }.compact)
+      fail_errors = [AuthenticationRequiredError, BadAuthenticationError, AuthenticationForbiddenError, FallbackError, SecurityError]
+      fail_errors << Gem::Requirement::BadRequirementError
+      fail_errors.concat(NET_ERRORS.map {|e| Gem::Net.const_get(e) })
     end.freeze
 
     class << self
@@ -79,6 +96,7 @@ module Bundler
     self.max_retries    = Bundler.settings[:retry] # How many retries for the API call
 
     def initialize(remote)
+      @cis = nil
       @remote = remote
 
       Socket.do_not_reverse_lookup = true
@@ -94,14 +112,17 @@ module Bundler
       spec -= [nil, "ruby", ""]
       spec_file_name = "#{spec.join "-"}.gemspec"
 
-      uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
-      if uri.scheme == "file"
-        Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(uri.path))
+      uri = Gem::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
+      spec = if uri.scheme == "file"
+        path = Gem::Util.correct_for_windows_path(uri.path)
+        Bundler.safe_load_marshal Bundler.rubygems.inflate(Gem.read_binary(path))
       elsif cached_spec_path = gemspec_cached_path(spec_file_name)
         Bundler.load_gemspec(cached_spec_path)
       else
-        Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
+        Bundler.safe_load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
       end
+      raise MarshalError, "is #{spec.inspect}" unless spec.is_a?(Gem::Specification)
+      spec
     rescue MarshalError
       raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
         "Your network or your gem server is probably having issues right now."
@@ -116,26 +137,13 @@ module Bundler
 
     # return the specs in the bundler format as an index
     def specs(gem_names, source)
-      old = Bundler.rubygems.sources
       index = Bundler::Index.new
 
-      if Bundler::Fetcher.disable_endpoint
-        @use_api = false
-        specs = fetchers.last.specs(gem_names)
-      else
-        specs = []
-        fetchers.shift until fetchers.first.available? || fetchers.empty?
-        fetchers.dup.each do |f|
-          break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
-          fetchers.delete(f)
-        end
-        @use_api = false if fetchers.none?(&:api_fetcher?)
-      end
-
-      specs.each do |name, version, platform, dependencies, metadata|
-        next if name == "bundler"
+      fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata|
         spec = if dependencies
-          EndpointSpecification.new(name, version, platform, dependencies, metadata)
+          EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es|
+            source.checksum_store.replace(es, es.checksum)
+          end
         else
           RemoteSpecification.new(name, version, platform, self)
         end
@@ -146,22 +154,8 @@ module Bundler
 
       index
     rescue CertificateFailureError
-      Bundler.ui.info "" if gem_names && use_api # newline after dots
+      Bundler.ui.info "" if gem_names && api_fetcher? # newline after dots
       raise
-    ensure
-      Bundler.rubygems.sources = old
-    end
-
-    def use_api
-      return @use_api if defined?(@use_api)
-
-      fetchers.shift until fetchers.first.available?
-
-      @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
-        false
-      else
-        fetchers.first.api_fetcher?
-      end
     end
 
     def user_agent
@@ -189,7 +183,7 @@ module Bundler
         agent << " ci/#{cis.join(",")}" if cis.any?
 
         # add a random ID so we can consolidate runs server-side
-        agent << " " << SecureRandom.hex(8)
+        agent << " " << Gem::SecureRandom.hex(8)
 
         # add any user agent strings set in the config
         extra_ua = Bundler.settings[:user_agent]
@@ -199,10 +193,6 @@ module Bundler
       end
     end
 
-    def fetchers
-      @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) }
-    end
-
     def http_proxy
       return unless uri = connection.proxy_uri
       uri.to_s
@@ -212,35 +202,67 @@ module Bundler
       "#<#{self.class}:0x#{object_id} uri=#{uri}>"
     end
 
-  private
+    def api_fetcher?
+      fetchers.first.api_fetcher?
+    end
+
+    def gem_remote_fetcher
+      @gem_remote_fetcher ||= begin
+        require_relative "fetcher/gem_remote_fetcher"
+        fetcher = GemRemoteFetcher.new Gem.configuration[:http_proxy]
+        fetcher.headers["User-Agent"] = user_agent
+        fetcher.headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
+        fetcher
+      end
+    end
+
+    private
+
+    def available_fetchers
+      if Bundler::Fetcher.disable_endpoint
+        [Index]
+      elsif remote_uri.scheme == "file"
+        Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API")
+        [Index]
+      else
+        [CompactIndex, Dependency, Index]
+      end
+    end
+
+    def fetchers
+      @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri, gem_remote_fetcher) }.drop_while {|f| !f.available? }
+    end
 
-    FETCHERS = [CompactIndex, Dependency, Index].freeze
+    def fetch_specs(gem_names)
+      fetchers.reject!(&:api_fetcher?) unless gem_names
+      fetchers.reject! do |f|
+        specs = f.specs(gem_names)
+        return specs if specs
+        true
+      end
+      []
+    end
 
     def cis
-      env_cis = {
-        "TRAVIS" => "travis",
-        "CIRCLECI" => "circle",
-        "SEMAPHORE" => "semaphore",
-        "JENKINS_URL" => "jenkins",
-        "BUILDBOX" => "buildbox",
-        "GO_SERVER_URL" => "go",
-        "SNAP_CI" => "snap",
-        "CI_NAME" => ENV["CI_NAME"],
-        "CI" => "ci"
-      }
-      env_cis.find_all {|env, _| ENV[env] }.map {|_, ci| ci }
+      @cis ||= Bundler::CIDetector.ci_strings
     end
 
     def connection
       @connection ||= begin
         needs_ssl = remote_uri.scheme == "https" ||
-          Bundler.settings[:ssl_verify_mode] ||
-          Bundler.settings[:ssl_client_cert]
-        raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)
+                    Bundler.settings[:ssl_verify_mode] ||
+                    Bundler.settings[:ssl_client_cert]
+        if needs_ssl
+          begin
+            require "openssl"
+          rescue StandardError, LoadError => e
+            raise SSLError.new(e.message)
+          end
+        end
 
-        con = PersistentHTTP.new "bundler", :ENV
-        if gem_proxy = Bundler.rubygems.configuration[:http_proxy]
-          con.proxy = URI.parse(gem_proxy) if gem_proxy != :no_proxy
+        con = Gem::Net::HTTP::Persistent.new name: "bundler", proxy: :ENV
+        if gem_proxy = Gem.configuration[:http_proxy]
+          con.proxy = Gem::URI.parse(gem_proxy) if gem_proxy != :no_proxy
         end
 
         if remote_uri.scheme == "https"
@@ -250,8 +272,8 @@ module Bundler
         end
 
         ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
-          (Bundler.rubygems.configuration.ssl_client_cert if
-            Bundler.rubygems.configuration.respond_to?(:ssl_client_cert))
+                          (Gem.configuration.ssl_client_cert if
+                            Gem.configuration.respond_to?(:ssl_client_cert))
         if ssl_client_cert
           pem = File.read(ssl_client_cert)
           con.cert = OpenSSL::X509::Certificate.new(pem)
@@ -269,22 +291,21 @@ module Bundler
     # cached gem specification path, if one exists
     def gemspec_cached_path(spec_file_name)
       paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
-      paths = paths.select {|path| File.file? path }
-      paths.first
+      paths.find {|path| File.file? path }
     end
 
     HTTP_ERRORS = [
-      Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH,
+      Gem::Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH,
       Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EAGAIN,
-      Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-      PersistentHTTP::Error, Zlib::BufError, Errno::EHOSTUNREACH
+      Gem::Net::HTTPBadResponse, Gem::Net::HTTPHeaderSyntaxError, Gem::Net::ProtocolError,
+      Gem::Net::HTTP::Persistent::Error, Zlib::BufError, Errno::EHOSTUNREACH
     ].freeze
 
     def bundler_cert_store
       store = OpenSSL::X509::Store.new
       ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
-        (Bundler.rubygems.configuration.ssl_ca_cert if
-          Bundler.rubygems.configuration.respond_to?(:ssl_ca_cert))
+                    (Gem.configuration.ssl_ca_cert if
+                      Gem.configuration.respond_to?(:ssl_ca_cert))
       if ssl_ca_cert
         if File.directory? ssl_ca_cert
           store.add_path ssl_ca_cert
@@ -293,14 +314,12 @@ module Bundler
         end
       else
         store.set_default_paths
-        certs = File.expand_path("../ssl_certs/*/*.pem", __FILE__)
-        Dir.glob(certs).each {|c| store.add_file c }
+        require "rubygems/request"
+        Gem::Request.get_cert_files.each {|c| store.add_file c }
       end
       store
     end
 
-  private
-
     def remote_uri
       @remote.uri
     end

data/lib/bundler/force_platform.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Bundler
+  module ForcePlatform
+    # The `:force_ruby_platform` value used by dependencies for resolution, and
+    # by locked specifications for materialization is `false` by default, except
+    # for TruffleRuby. TruffleRuby generally needs to force the RUBY platform
+    # variant unless the name is explicitly allowlisted.
+
+    def default_force_ruby_platform
+      return false unless RUBY_ENGINE == "truffleruby"
+
+      !Gem::Platform::REUSE_AS_BINARY_ON_TRUFFLERUBY.include?(name)
+    end
+  end
+end