bundler 1.11.1 → 2.2.6

data/lib/bundler/feature_flag.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Bundler
+  class FeatureFlag
+    def self.settings_flag(flag, &default)
+      unless Bundler::Settings::BOOL_KEYS.include?(flag.to_s)
+        raise "Cannot use `#{flag}` as a settings feature flag since it isn't a bool key"
+      end
+
+      settings_method("#{flag}?", flag, &default)
+    end
+    private_class_method :settings_flag
+
+    def self.settings_option(key, &default)
+      settings_method(key, key, &default)
+    end
+    private_class_method :settings_option
+
+    def self.settings_method(name, key, &default)
+      define_method(name) do
+        value = Bundler.settings[key]
+        value = instance_eval(&default) if value.nil?
+        value
+      end
+    end
+    private_class_method :settings_method
+
+    (1..10).each {|v| define_method("bundler_#{v}_mode?") { major_version >= v } }
+
+    settings_flag(:allow_bundler_dependency_conflicts) { bundler_3_mode? }
+    settings_flag(:allow_offline_install) { bundler_3_mode? }
+    settings_flag(:auto_clean_without_path) { bundler_3_mode? }
+    settings_flag(:cache_all) { bundler_3_mode? }
+    settings_flag(:default_install_uses_path) { bundler_3_mode? }
+    settings_flag(:deployment_means_frozen) { bundler_3_mode? }
+    settings_flag(:disable_multisource) { bundler_3_mode? }
+    settings_flag(:forget_cli_options) { bundler_3_mode? }
+    settings_flag(:global_gem_cache) { bundler_3_mode? }
+    settings_flag(:only_update_to_newer_versions) { bundler_3_mode? }
+    settings_flag(:path_relative_to_cwd) { bundler_3_mode? }
+    settings_flag(:plugins) { @bundler_version >= Gem::Version.new("1.14") }
+    settings_flag(:print_only_version_number) { bundler_3_mode? }
+    settings_flag(:setup_makes_kernel_gem_public) { !bundler_3_mode? }
+    settings_flag(:suppress_install_using_messages) { bundler_3_mode? }
+    settings_flag(:unlock_source_unlocks_spec) { !bundler_3_mode? }
+    settings_flag(:update_requires_all_flag) { bundler_4_mode? }
+    settings_flag(:use_gem_version_promoter_for_major_updates) { bundler_3_mode? }
+
+    settings_option(:default_cli_command) { bundler_3_mode? ? :cli_help : :install }
+
+    def initialize(bundler_version)
+      @bundler_version = Gem::Version.create(bundler_version)
+    end
+
+    def major_version
+      @bundler_version.segments.first
+    end
+    private :major_version
+  end
+end

data/lib/bundler/fetcher.rb

@@ -1,22 +1,29 @@
-require "bundler/vendored_persistent"
+# frozen_string_literal: true
+
+require_relative "vendored_persistent"
 require "cgi"
 require "securerandom"
 require "zlib"
+require "rubygems/request"
 
 module Bundler
   # Handles all the fetching with the rubygems server
   class Fetcher
-    autoload :Downloader, "bundler/fetcher/downloader"
-    autoload :Dependency, "bundler/fetcher/dependency"
-    autoload :Index, "bundler/fetcher/index"
+    autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__)
+    autoload :Downloader, File.expand_path("fetcher/downloader", __dir__)
+    autoload :Dependency, File.expand_path("fetcher/dependency", __dir__)
+    autoload :Index, File.expand_path("fetcher/index", __dir__)
 
     # This error is raised when it looks like the network is down
     class NetworkDownError < HTTPError; end
+    # This error is raised if we should rate limit our requests to the API
+    class TooManyRequestsError < HTTPError; end
     # This error is raised if the API returns a 413 (only printed in verbose)
     class FallbackError < HTTPError; end
     # This is the error raised if OpenSSL fails the cert verification
     class CertificateFailureError < HTTPError
       def initialize(remote_uri)
+        remote_uri = filter_uri(remote_uri)
         super "Could not verify the SSL certificate for #{remote_uri}.\nThere" \
           " is a chance you are experiencing a man-in-the-middle attack, but" \
           " most likely your system doesn't have the CA certificates needed" \
@@ -37,14 +44,16 @@ module Bundler
     # This error is raised if HTTP authentication is required, but not provided.
     class AuthenticationRequiredError < HTTPError
       def initialize(remote_uri)
+        remote_uri = filter_uri(remote_uri)
         super "Authentication is required for #{remote_uri}.\n" \
           "Please supply credentials for this source. You can do this by running:\n" \
-          " bundle config #{remote_uri} username:password"
+          " bundle config set --global #{remote_uri} username:password"
       end
     end
     # This error is raised if HTTP authentication is provided, but incorrect.
     class BadAuthenticationError < HTTPError
       def initialize(remote_uri)
+        remote_uri = filter_uri(remote_uri)
         super "Bad username or password for #{remote_uri}.\n" \
           "Please double-check your credentials and correct them."
       end
@@ -52,10 +61,17 @@ module Bundler
 
     # Exceptions classes that should bypass retry attempts. If your password didn't work the
     # first time, it's not going to the third time.
-    AUTH_ERRORS = [AuthenticationRequiredError, BadAuthenticationError, Net::HTTPBadGateway,
-                   Net::HTTPBadRequest, Net::HTTPForbidden, Net::HTTPMethodNotAllowed,
-                   Net::HTTPMovedPermanently, Net::HTTPNotImplemented, Net::HTTPNotFound,
-                   Net::HTTPRequestEntityTooLarge, Net::HTTPNoContent]
+    NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency,
+                  :HTTPForbidden, :HTTPInsufficientStorage, :HTTPMethodNotAllowed,
+                  :HTTPMovedPermanently, :HTTPNoContent, :HTTPNotFound,
+                  :HTTPNotImplemented, :HTTPPreconditionFailed, :HTTPRequestEntityTooLarge,
+                  :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity,
+                  :HTTPUnsupportedMediaType, :HTTPVersionNotSupported].freeze
+    FAIL_ERRORS = begin
+      fail_errors = [AuthenticationRequiredError, BadAuthenticationError, FallbackError]
+      fail_errors << Gem::Requirement::BadRequirementError if defined?(Gem::Requirement::BadRequirementError)
+      fail_errors.concat(NET_ERRORS.map {|e| SharedHelpers.const_get_safely(e, Net) }.compact)
+    end.freeze
 
     class << self
       attr_accessor :disable_endpoint, :api_timeout, :redirect_limit, :max_retries
@@ -81,13 +97,14 @@ module Bundler
       spec -= [nil, "ruby", ""]
       spec_file_name = "#{spec.join "-"}.gemspec"
 
-      uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
+      uri = Bundler::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
       if uri.scheme == "file"
-        Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
+        path = Bundler.rubygems.correct_for_windows_path(uri.path)
+        Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(path))
       elsif cached_spec_path = gemspec_cached_path(spec_file_name)
         Bundler.load_gemspec(cached_spec_path)
       else
-        Bundler.load_marshal Gem.inflate(downloader.fetch uri)
+        Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
       end
     rescue MarshalError
       raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
@@ -96,7 +113,7 @@ module Bundler
 
     # return the specs in the bundler format as an index with retries
     def specs_with_retry(gem_names, source)
-      Bundler::Retry.new("fetcher").attempts do
+      Bundler::Retry.new("fetcher", FAIL_ERRORS).attempts do
         specs(gem_names, source)
       end
     end
@@ -106,20 +123,25 @@ module Bundler
       old = Bundler.rubygems.sources
       index = Bundler::Index.new
 
-      specs = {}
-      fetchers.dup.each do |f|
-        break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
-        fetchers.delete(f)
+      if Bundler::Fetcher.disable_endpoint
+        @use_api = false
+        specs = fetchers.last.specs(gem_names)
+      else
+        specs = []
+        fetchers.shift until fetchers.first.available? || fetchers.empty?
+        fetchers.dup.each do |f|
+          break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
+          fetchers.delete(f)
+        end
+        @use_api = false if fetchers.none?(&:api_fetcher?)
       end
-      @use_api = false if fetchers.none?(&:api_fetcher?)
 
-      specs[remote_uri].each do |name, version, platform, dependencies|
+      specs.each do |name, version, platform, dependencies, metadata|
         next if name == "bundler"
-        spec = nil
-        if dependencies
-          spec = EndpointSpecification.new(name, version, platform, dependencies)
+        spec = if dependencies
+          EndpointSpecification.new(name, version, platform, dependencies, metadata)
         else
-          spec = RemoteSpecification.new(name, version, platform, self)
+          RemoteSpecification.new(name, version, platform, self)
         end
         spec.source = source
         spec.remote = @remote
@@ -137,32 +159,33 @@ module Bundler
     def use_api
       return @use_api if defined?(@use_api)
 
-      if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
-        @use_api = false
+      fetchers.shift until fetchers.first.available?
+
+      @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
+        false
       else
-        fetchers.reject! {|f| f.api_fetcher? && !f.api_available? }
-        @use_api = fetchers.any?(&:api_fetcher?)
+        fetchers.first.api_fetcher?
       end
     end
 
     def user_agent
       @user_agent ||= begin
-        ruby = Bundler.ruby_version
+        ruby = Bundler::RubyVersion.system
 
-        agent = "bundler/#{Bundler::VERSION}"
+        agent = String.new("bundler/#{Bundler::VERSION}")
         agent << " rubygems/#{Gem::VERSION}"
-        agent << " ruby/#{ruby.version}"
+        agent << " ruby/#{ruby.versions_string(ruby.versions)}"
         agent << " (#{ruby.host})"
         agent << " command/#{ARGV.first}"
 
         if ruby.engine != "ruby"
           # engine_version raises on unknown engines
           engine_version = begin
-                             ruby.engine_version
-                           rescue
+                             ruby.engine_versions
+                           rescue RuntimeError
                              "???"
                            end
-          agent << " #{ruby.engine}/#{engine_version}"
+          agent << " #{ruby.engine}/#{ruby.versions_string(engine_version)}"
         end
 
         agent << " options/#{Bundler.settings.all.join(",")}"
@@ -185,18 +208,17 @@ module Bundler
     end
 
     def http_proxy
-      if uri = connection.proxy_uri
-        uri.to_s
-      end
+      return unless uri = connection.proxy_uri
+      uri.to_s
     end
 
     def inspect
       "#<#{self.class}:0x#{object_id} uri=#{uri}>"
     end
 
-  private
+    private
 
-    FETCHERS = [Dependency, Index]
+    FETCHERS = [CompactIndex, Dependency, Index].freeze
 
     def cis
       env_cis = {
@@ -207,8 +229,9 @@ module Bundler
         "BUILDBOX" => "buildbox",
         "GO_SERVER_URL" => "go",
         "SNAP_CI" => "snap",
+        "GITLAB_CI" => "gitlab",
         "CI_NAME" => ENV["CI_NAME"],
-        "CI" => "ci"
+        "CI" => "ci",
       }
       env_cis.find_all {|env, _| ENV[env] }.map {|_, ci| ci }
     end
@@ -220,9 +243,9 @@ module Bundler
           Bundler.settings[:ssl_client_cert]
         raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)
 
-        con = Net::HTTP::Persistent.new "bundler", :ENV
+        con = PersistentHTTP.new :name => "bundler", :proxy => :ENV
         if gem_proxy = Bundler.rubygems.configuration[:http_proxy]
-          con.proxy = URI.parse(gem_proxy)
+          con.proxy = Bundler::URI.parse(gem_proxy) if gem_proxy != :no_proxy
         end
 
         if remote_uri.scheme == "https"
@@ -231,13 +254,17 @@ module Bundler
           con.cert_store = bundler_cert_store
         end
 
-        if Bundler.settings[:ssl_client_cert]
-          pem = File.read(Bundler.settings[:ssl_client_cert])
+        ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
+          (Bundler.rubygems.configuration.ssl_client_cert if
+            Bundler.rubygems.configuration.respond_to?(:ssl_client_cert))
+        if ssl_client_cert
+          pem = File.read(ssl_client_cert)
           con.cert = OpenSSL::X509::Certificate.new(pem)
           con.key  = OpenSSL::PKey::RSA.new(pem)
         end
 
         con.read_timeout = Fetcher.api_timeout
+        con.open_timeout = Fetcher.api_timeout
         con.override_headers["User-Agent"] = user_agent
         con.override_headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
         con
@@ -255,26 +282,28 @@ module Bundler
       Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH,
       Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EAGAIN,
       Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-      Net::HTTP::Persistent::Error, Zlib::BufError
-    ]
+      PersistentHTTP::Error, Zlib::BufError, Errno::EHOSTUNREACH
+    ].freeze
 
     def bundler_cert_store
       store = OpenSSL::X509::Store.new
-      if Bundler.settings[:ssl_ca_cert]
-        if File.directory? Bundler.settings[:ssl_ca_cert]
-          store.add_path Bundler.settings[:ssl_ca_cert]
+      ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
+        (Bundler.rubygems.configuration.ssl_ca_cert if
+          Bundler.rubygems.configuration.respond_to?(:ssl_ca_cert))
+      if ssl_ca_cert
+        if File.directory? ssl_ca_cert
+          store.add_path ssl_ca_cert
         else
-          store.add_file Bundler.settings[:ssl_ca_cert]
+          store.add_file ssl_ca_cert
         end
       else
         store.set_default_paths
-        certs = File.expand_path("../ssl_certs/*.pem", __FILE__)
-        Dir.glob(certs).each {|c| store.add_file c }
+        Gem::Request.get_cert_files.each {|c| store.add_file c }
       end
       store
     end
 
-  private
+    private
 
     def remote_uri
       @remote.uri

data/lib/bundler/fetcher/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bundler
   class Fetcher
     class Base
@@ -20,7 +22,7 @@ module Bundler
         @fetch_uri ||= begin
           if remote_uri.host == "rubygems.org"
             uri = remote_uri.dup
-            uri.host = "bundler.rubygems.org"
+            uri.host = "index.rubygems.org"
             uri
           else
             remote_uri
@@ -28,13 +30,23 @@ module Bundler
         end
       end
 
-      def api_available?
-        api_fetcher?
+      def available?
+        true
       end
 
       def api_fetcher?
         false
       end
+
+      private
+
+      def log_specs(debug_msg)
+        if Bundler.ui.debug?
+          Bundler.ui.debug debug_msg
+        else
+          Bundler.ui.info ".", false
+        end
+      end
     end
   end
 end

data/lib/bundler/fetcher/compact_index.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../worker"
+
+module Bundler
+  autoload :CompactIndexClient, File.expand_path("../compact_index_client", __dir__)
+
+  class Fetcher
+    class CompactIndex < Base
+      def self.compact_index_request(method_name)
+        method = instance_method(method_name)
+        undef_method(method_name)
+        define_method(method_name) do |*args, &blk|
+          begin
+            method.bind(self).call(*args, &blk)
+          rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e
+            raise HTTPError, e.message
+          rescue AuthenticationRequiredError
+            # Fail since we got a 401 from the server.
+            raise
+          rescue HTTPError => e
+            Bundler.ui.trace(e)
+            nil
+          end
+        end
+      end
+
+      def specs(gem_names)
+        specs_for_names(gem_names)
+      end
+      compact_index_request :specs
+
+      def specs_for_names(gem_names)
+        gem_info = []
+        complete_gems = []
+        remaining_gems = gem_names.dup
+
+        until remaining_gems.empty?
+          log_specs "Looking up gems #{remaining_gems.inspect}"
+
+          deps = begin
+                   parallel_compact_index_client.dependencies(remaining_gems)
+                 rescue TooManyRequestsError
+                   @bundle_worker.stop if @bundle_worker
+                   @bundle_worker = nil # reset it.  Not sure if necessary
+                   serial_compact_index_client.dependencies(remaining_gems)
+                 end
+          next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq
+          deps.each {|dep| gem_info << dep }
+          complete_gems.concat(deps.map(&:first)).uniq!
+          remaining_gems = next_gems - complete_gems
+        end
+        @bundle_worker.stop if @bundle_worker
+        @bundle_worker = nil # reset it.  Not sure if necessary
+
+        gem_info
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        contents = compact_index_client.spec(*spec)
+        return nil if contents.nil?
+        contents.unshift(spec.first)
+        contents[3].map! {|d| Gem::Dependency.new(*d) }
+        EndpointSpecification.new(*contents)
+      end
+      compact_index_request :fetch_spec
+
+      def available?
+        return nil unless SharedHelpers.md5_available?
+        user_home = Bundler.user_home
+        return nil unless user_home.directory? && user_home.writable?
+        # Read info file checksums out of /versions, so we can know if gems are up to date
+        fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums!
+      rescue CompactIndexClient::Updater::MisMatchedChecksumError => e
+        Bundler.ui.debug(e.message)
+        nil
+      end
+      compact_index_request :available?
+
+      def api_fetcher?
+        true
+      end
+
+      private
+
+      def compact_index_client
+        @compact_index_client ||=
+          SharedHelpers.filesystem_access(cache_path) do
+            CompactIndexClient.new(cache_path, client_fetcher)
+          end
+      end
+
+      def parallel_compact_index_client
+        compact_index_client.execution_mode = lambda do |inputs, &blk|
+          func = lambda {|object, _index| blk.call(object) }
+          worker = bundle_worker(func)
+          inputs.each {|input| worker.enq(input) }
+          inputs.map { worker.deq }
+        end
+
+        compact_index_client
+      end
+
+      def serial_compact_index_client
+        compact_index_client.sequential_execution_mode!
+        compact_index_client
+      end
+
+      def bundle_worker(func = nil)
+        @bundle_worker ||= begin
+          worker_name = "Compact Index (#{display_uri.host})"
+          Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func)
+        end
+        @bundle_worker.tap do |worker|
+          worker.instance_variable_set(:@func, func) if func
+        end
+      end
+
+      def cache_path
+        Bundler.user_cache.join("compact_index", remote.cache_slug)
+      end
+
+      def client_fetcher
+        ClientFetcher.new(self, Bundler.ui)
+      end
+
+      ClientFetcher = Struct.new(:fetcher, :ui) do
+        def call(path, headers)
+          fetcher.downloader.fetch(fetcher.fetch_uri + path, headers)
+        rescue NetworkDownError => e
+          raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"]
+          ui.warn "Using the cached data for the new index because of a network error: #{e}"
+          Net::HTTPNotModified.new(nil, nil, nil)
+        end
+      end
+    end
+  end
+end