bundler 1.9.0 → 1.17.3

data/lib/bundler/fetcher/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Base
+      attr_reader :downloader
+      attr_reader :display_uri
+      attr_reader :remote
+
+      def initialize(downloader, remote, display_uri)
+        raise "Abstract class" if self.class == Base
+        @downloader = downloader
+        @remote = remote
+        @display_uri = display_uri
+      end
+
+      def remote_uri
+        @remote.uri
+      end
+
+      def fetch_uri
+        @fetch_uri ||= begin
+          if remote_uri.host == "rubygems.org"
+            uri = remote_uri.dup
+            uri.host = "index.rubygems.org"
+            uri
+          else
+            remote_uri
+          end
+        end
+      end
+
+      def available?
+        true
+      end
+
+      def api_fetcher?
+        false
+      end
+
+    private
+
+      def log_specs(debug_msg)
+        if Bundler.ui.debug?
+          Bundler.ui.debug debug_msg
+        else
+          Bundler.ui.info ".", false
+        end
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/compact_index.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "bundler/worker"
+
+module Bundler
+  autoload :CompactIndexClient, "bundler/compact_index_client"
+
+  class Fetcher
+    class CompactIndex < Base
+      def self.compact_index_request(method_name)
+        method = instance_method(method_name)
+        undef_method(method_name)
+        define_method(method_name) do |*args, &blk|
+          begin
+            method.bind(self).call(*args, &blk)
+          rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e
+            raise HTTPError, e.message
+          rescue AuthenticationRequiredError
+            # Fail since we got a 401 from the server.
+            raise
+          rescue HTTPError => e
+            Bundler.ui.trace(e)
+            nil
+          end
+        end
+      end
+
+      def specs(gem_names)
+        specs_for_names(gem_names)
+      end
+      compact_index_request :specs
+
+      def specs_for_names(gem_names)
+        gem_info = []
+        complete_gems = []
+        remaining_gems = gem_names.dup
+
+        until remaining_gems.empty?
+          log_specs "Looking up gems #{remaining_gems.inspect}"
+
+          deps = compact_index_client.dependencies(remaining_gems)
+          next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq
+          deps.each {|dep| gem_info << dep }
+          complete_gems.concat(deps.map(&:first)).uniq!
+          remaining_gems = next_gems - complete_gems
+        end
+        @bundle_worker.stop if @bundle_worker
+        @bundle_worker = nil # reset it.  Not sure if necessary
+
+        gem_info
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        contents = compact_index_client.spec(*spec)
+        return nil if contents.nil?
+        contents.unshift(spec.first)
+        contents[3].map! {|d| Gem::Dependency.new(*d) }
+        EndpointSpecification.new(*contents)
+      end
+      compact_index_request :fetch_spec
+
+      def available?
+        return nil unless SharedHelpers.md5_available?
+        user_home = Bundler.user_home
+        return nil unless user_home.directory? && user_home.writable?
+        # Read info file checksums out of /versions, so we can know if gems are up to date
+        fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums!
+      rescue CompactIndexClient::Updater::MisMatchedChecksumError => e
+        Bundler.ui.debug(e.message)
+        nil
+      end
+      compact_index_request :available?
+
+      def api_fetcher?
+        true
+      end
+
+    private
+
+      def compact_index_client
+        @compact_index_client ||= begin
+          SharedHelpers.filesystem_access(cache_path) do
+            CompactIndexClient.new(cache_path, client_fetcher)
+          end.tap do |client|
+            client.in_parallel = lambda do |inputs, &blk|
+              func = lambda {|object, _index| blk.call(object) }
+              worker = bundle_worker(func)
+              inputs.each {|input| worker.enq(input) }
+              inputs.map { worker.deq }
+            end
+          end
+        end
+      end
+
+      def bundle_worker(func = nil)
+        @bundle_worker ||= begin
+          worker_name = "Compact Index (#{display_uri.host})"
+          Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func)
+        end
+        @bundle_worker.tap do |worker|
+          worker.instance_variable_set(:@func, func) if func
+        end
+      end
+
+      def cache_path
+        Bundler.user_cache.join("compact_index", remote.cache_slug)
+      end
+
+      def client_fetcher
+        ClientFetcher.new(self, Bundler.ui)
+      end
+
+      ClientFetcher = Struct.new(:fetcher, :ui) do
+        def call(path, headers)
+          fetcher.downloader.fetch(fetcher.fetch_uri + path, headers)
+        rescue NetworkDownError => e
+          raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"]
+          ui.warn "Using the cached data for the new index because of a network error: #{e}"
+          Net::HTTPNotModified.new(nil, nil, nil)
+        end
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/dependency.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "cgi"
+
+module Bundler
+  class Fetcher
+    class Dependency < Base
+      def available?
+        @available ||= fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri)
+      rescue NetworkDownError => e
+        raise HTTPError, e.message
+      rescue AuthenticationRequiredError
+        # Fail since we got a 401 from the server.
+        raise
+      rescue HTTPError
+        false
+      end
+
+      def api_fetcher?
+        true
+      end
+
+      def specs(gem_names, full_dependency_list = [], last_spec_list = [])
+        query_list = gem_names.uniq - full_dependency_list
+
+        log_specs "Query List: #{query_list.inspect}"
+
+        return last_spec_list if query_list.empty?
+
+        spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do
+          dependency_specs(query_list)
+        end
+
+        returned_gems = spec_list.map(&:first).uniq
+        specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
+      rescue MarshalError
+        Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+        Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
+        nil
+      rescue HTTPError, GemspecError
+        Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+        Bundler.ui.debug "could not fetch from the dependency API\nit's suggested to retry using the full index via `bundle install --full-index`"
+        nil
+      end
+
+      def dependency_specs(gem_names)
+        Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}"
+
+        gem_list = unmarshalled_dep_gems(gem_names)
+        get_formatted_specs_and_deps(gem_list)
+      end
+
+      def unmarshalled_dep_gems(gem_names)
+        gem_list = []
+        gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
+          marshalled_deps = downloader.fetch(dependency_api_uri(names)).body
+          gem_list.concat(Bundler.load_marshal(marshalled_deps))
+        end
+        gem_list
+      end
+
+      def get_formatted_specs_and_deps(gem_list)
+        deps_list = []
+        spec_list = []
+
+        gem_list.each do |s|
+          deps_list.concat(s[:dependencies].map(&:first))
+          deps = s[:dependencies].map {|n, d| [n, d.split(", ")] }
+          spec_list.push([s[:name], s[:number], s[:platform], deps])
+        end
+        [spec_list, deps_list]
+      end
+
+      def dependency_api_uri(gem_names = [])
+        uri = fetch_uri + "api/v1/dependencies"
+        uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any?
+        uri
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/downloader.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Downloader
+      attr_reader :connection
+      attr_reader :redirect_limit
+
+      def initialize(connection, redirect_limit)
+        @connection = connection
+        @redirect_limit = redirect_limit
+      end
+
+      def fetch(uri, headers = {}, counter = 0)
+        raise HTTPError, "Too many redirects" if counter >= redirect_limit
+
+        response = request(uri, headers)
+        Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}")
+
+        case response
+        when Net::HTTPSuccess, Net::HTTPNotModified
+          response
+        when Net::HTTPRedirection
+          new_uri = URI.parse(response["location"])
+          if new_uri.host == uri.host
+            new_uri.user = uri.user
+            new_uri.password = uri.password
+          end
+          fetch(new_uri, headers, counter + 1)
+        when Net::HTTPRequestedRangeNotSatisfiable
+          new_headers = headers.dup
+          new_headers.delete("Range")
+          new_headers["Accept-Encoding"] = "gzip"
+          fetch(uri, new_headers)
+        when Net::HTTPRequestEntityTooLarge
+          raise FallbackError, response.body
+        when Net::HTTPUnauthorized
+          raise AuthenticationRequiredError, uri.host
+        when Net::HTTPNotFound
+          raise FallbackError, "Net::HTTPNotFound"
+        else
+          raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
+        end
+      end
+
+      def request(uri, headers)
+        validate_uri_scheme!(uri)
+
+        Bundler.ui.debug "HTTP GET #{uri}"
+        req = Net::HTTP::Get.new uri.request_uri, headers
+        if uri.user
+          user = CGI.unescape(uri.user)
+          password = uri.password ? CGI.unescape(uri.password) : nil
+          req.basic_auth(user, password)
+        end
+        connection.request(uri, req)
+      rescue NoMethodError => e
+        raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet }
+        raise LoadError.new("cannot load such file -- openssl")
+      rescue OpenSSL::SSL::SSLError
+        raise CertificateFailureError.new(uri)
+      rescue *HTTP_ERRORS => e
+        Bundler.ui.trace e
+        case e.message
+        when /host down:/, /getaddrinfo: nodename nor servname provided/
+          raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \
+            "connection and try again."
+        else
+          raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \
+            " (#{e})"
+        end
+      end
+
+    private
+
+      def validate_uri_scheme!(uri)
+        return if uri.scheme =~ /\Ahttps?\z/
+        raise InvalidOption,
+          "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \
+          "Did you mean `http` or `https`?"
+      end
+    end
+  end
+end

data/lib/bundler/fetcher/index.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    class Index < Base
+      def specs(_gem_names)
+        Bundler.rubygems.fetch_all_remote_specs(remote)
+      rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e
+        case e.message
+        when /certificate verify failed/
+          raise CertificateFailureError.new(display_uri)
+        when /401/
+          raise AuthenticationRequiredError, remote_uri
+        when /403/
+          raise BadAuthenticationError, remote_uri if remote_uri.userinfo
+          raise AuthenticationRequiredError, remote_uri
+        else
+          Bundler.ui.trace e
+          raise HTTPError, "Could not fetch specs from #{display_uri}"
+        end
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        spec_file_name = "#{spec.join "-"}.gemspec"
+
+        uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
+        if uri.scheme == "file"
+          Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(uri.path))
+        elsif cached_spec_path = gemspec_cached_path(spec_file_name)
+          Bundler.load_gemspec(cached_spec_path)
+        else
+          Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
+        end
+      rescue MarshalError
+        raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
+          "Your network or your gem server is probably having issues right now."
+      end
+
+    private
+
+      # cached gem specification path, if one exists
+      def gemspec_cached_path(spec_file_name)
+        paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
+        paths.find {|path| File.file? path }
+      end
+    end
+  end
+end

data/lib/bundler/friendly_errors.rb

@@ -1,76 +1,131 @@
 # encoding: utf-8
+# frozen_string_literal: true
+
 require "cgi"
 require "bundler/vendored_thor"
 
 module Bundler
-  def self.with_friendly_errors
-    yield
-  rescue Bundler::BundlerError => e
-    Bundler.ui.error e.message, :wrap => true
-    Bundler.ui.trace e
-    exit e.status_code
-  rescue Thor::AmbiguousTaskError => e
-    Bundler.ui.error e.message
-    exit 15
-  rescue Thor::UndefinedTaskError => e
-    Bundler.ui.error e.message
-    exit 15
-  rescue Thor::Error => e
-    Bundler.ui.error e.message
-    exit 1
-  rescue LoadError => e
-    raise e unless e.message =~ /cannot load such file -- openssl|openssl.so|libcrypto.so/
-    Bundler.ui.error "\nCould not load OpenSSL."
-    Bundler.ui.warn <<-WARN, :wrap => true
-      You must recompile Ruby with OpenSSL support or change the sources in your \
-      Gemfile from 'https' to 'http'. Instructions for compiling with OpenSSL \
-      using RVM are available at http://rvm.io/packages/openssl.
-    WARN
-    Bundler.ui.trace e
-    exit 1
-  rescue Interrupt => e
-    Bundler.ui.error "\nQuitting..."
-    Bundler.ui.trace e
-    exit 1
-  rescue SystemExit => e
-    exit e.status
-  rescue Exception => e
-    request_issue_report_for(e)
-    exit 1
-  end
+  module FriendlyErrors
+  module_function
 
-  def self.request_issue_report_for(e)
-    Bundler.ui.info <<-EOS.gsub(/^ {6}/, '')
-      #{'――― ERROR REPORT TEMPLATE ―――――――――――――――――――――――――――――――――――――――――――――――――――――――'}
-      - What did you do?
-      - What did you expect to happen?
-      - What happened instead?
+    def log_error(error)
+      case error
+      when YamlSyntaxError
+        Bundler.ui.error error.message
+        Bundler.ui.trace error.orig_exception
+      when Dsl::DSLError, GemspecError
+        Bundler.ui.error error.message
+      when GemRequireError
+        Bundler.ui.error error.message
+        Bundler.ui.trace error.orig_exception, nil, true
+      when BundlerError
+        Bundler.ui.error error.message, :wrap => true
+        Bundler.ui.trace error
+      when Thor::Error
+        Bundler.ui.error error.message
+      when LoadError
+        raise error unless error.message =~ /cannot load such file -- openssl|openssl.so|libcrypto.so/
+        Bundler.ui.error "\nCould not load OpenSSL."
+        Bundler.ui.warn <<-WARN, :wrap => true
+          You must recompile Ruby with OpenSSL support or change the sources in your \
+          Gemfile from 'https' to 'http'. Instructions for compiling with OpenSSL \
+          using RVM are available at http://rvm.io/packages/openssl.
+        WARN
+        Bundler.ui.trace error
+      when Interrupt
+        Bundler.ui.error "\nQuitting..."
+        Bundler.ui.trace error
+      when Gem::InvalidSpecificationException
+        Bundler.ui.error error.message, :wrap => true
+      when SystemExit
+      when *[defined?(Java::JavaLang::OutOfMemoryError) && Java::JavaLang::OutOfMemoryError].compact
+        Bundler.ui.error "\nYour JVM has run out of memory, and Bundler cannot continue. " \
+          "You can decrease the amount of memory Bundler needs by removing gems from your Gemfile, " \
+          "especially large gems. (Gems can be as large as hundreds of megabytes, and Bundler has to read those files!). " \
+          "Alternatively, you can increase the amount of memory the JVM is able to use by running Bundler with jruby -J-Xmx1024m -S bundle (JRuby defaults to 500MB)."
+      else request_issue_report_for(error)
+      end
+    rescue
+      raise error
+    end
 
-      Error details
+    def exit_status(error)
+      case error
+      when BundlerError then error.status_code
+      when Thor::Error then 15
+      when SystemExit then error.status
+      else 1
+      end
+    end
 
-          #{e.class}: #{e.message}
-          #{e.backtrace.join("\n          ")}
+    def request_issue_report_for(e)
+      Bundler.ui.info <<-EOS.gsub(/^ {8}/, "")
+        --- ERROR REPORT TEMPLATE -------------------------------------------------------
+        # Error Report
 
-      #{Bundler::Env.new.report(:print_gemfile => false).gsub(/\n/, "\n      ").strip}
-      #{'――― TEMPLATE END ――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――'}
+        ## Questions
 
-    EOS
+        Please fill out answers to these questions, it'll help us figure out
+        why things are going wrong.
 
-    Bundler.ui.error "Unfortunately, an unexpected error occurred, and Bundler cannot continue."
+        - **What did you do?**
 
-    Bundler.ui.warn <<-EOS.gsub(/^ {6}/, '')
+          I ran the command `#{$PROGRAM_NAME} #{ARGV.join(" ")}`
 
-      First, try this link to see if there are any existing issue reports for this error:
-      #{issues_url(e)}
+        - **What did you expect to happen?**
 
-      If there aren't any reports for this error yet, please create copy and paste the report template above into a new issue. Don't forget to anonymize any private data! The new issue form is located at:
-      https://github.com/bundler/bundler/issues/new
-    EOS
-  end
+          I expected Bundler to...
+
+        - **What happened instead?**
+
+          Instead, what happened was...
+
+        - **Have you tried any solutions posted on similar issues in our issue tracker, stack overflow, or google?**
+
+          I tried...
+
+        - **Have you read our issues document, https://github.com/bundler/bundler/blob/master/doc/contributing/ISSUES.md?**
+
+          ...
 
-  def self.issues_url(exception)
-    'https://github.com/bundler/bundler/search?q=' \
-    "#{CGI.escape(exception.message)}&type=Issues"
+        ## Backtrace
+
+        ```
+        #{e.class}: #{e.message}
+          #{e.backtrace && e.backtrace.join("\n          ").chomp}
+        ```
+
+        #{Bundler::Env.report}
+        --- TEMPLATE END ----------------------------------------------------------------
+
+      EOS
+
+      Bundler.ui.error "Unfortunately, an unexpected error occurred, and Bundler cannot continue."
+
+      Bundler.ui.warn <<-EOS.gsub(/^ {8}/, "")
+
+        First, try this link to see if there are any existing issue reports for this error:
+        #{issues_url(e)}
+
+        If there aren't any reports for this error yet, please create copy and paste the report template above into a new issue. Don't forget to anonymize any private data! The new issue form is located at:
+        https://github.com/bundler/bundler/issues/new
+      EOS
+    end
+
+    def issues_url(exception)
+      message = exception.message.lines.first.tr(":", " ").chomp
+      message = message.split("-").first if exception.is_a?(Errno)
+      "https://github.com/bundler/bundler/search?q=" \
+        "#{CGI.escape(message)}&type=Issues"
+    end
   end
 
+  def self.with_friendly_errors
+    yield
+  rescue SignalException
+    raise
+  rescue Exception => e
+    FriendlyErrors.log_error(e)
+    exit FriendlyErrors.exit_status(e)
+  end
 end