rubygems-update 4.0.21 → 4.1.0.beta1

data/lib/rubygems/remote_fetcher.rb

@@ -47,6 +47,10 @@ class Gem::RemoteFetcher
   end
   deprecate_constant(:UnknownHostError)
 
+  # Schemes fetched over the network, as opposed to copied from a local path.
+  REMOTE_SCHEMES = %w[http https s3].freeze
+  private_constant :REMOTE_SCHEMES
+
   @fetcher = nil
 
   ##
@@ -111,17 +115,29 @@ class Gem::RemoteFetcher
   # always replaced.
 
   def download(spec, source_uri, install_dir = Gem.dir)
+    gem_file_name = File.basename spec.cache_file
+
+    source_uri = Gem::Uri.new(source_uri)
+
+    scheme = source_uri.scheme
+
+    # Gem::URI.parse gets confused by MS Windows paths with forward slashes.
+    scheme = nil if /^[a-z]$/i.match?(scheme)
+
+    remote_source = REMOTE_SCHEMES.include?(scheme)
+
     install_cache_dir = File.join install_dir, "cache"
     cache_dir =
-      if Dir.pwd == install_dir # see fetch_command
+      if File.identical?(".", install_dir) # gem fetch asks for it this way
         install_dir
+      elsif Gem.configuration.global_gem_cache && remote_source && ensure_writable_cache_dir(Gem.global_gem_cache_path)
+        Gem.global_gem_cache_path
       elsif File.writable?(install_cache_dir) || (File.writable?(install_dir) && !File.exist?(install_cache_dir))
         install_cache_dir
       else
         File.join Gem.user_dir, "cache"
       end
 
-    gem_file_name = File.basename spec.cache_file
     local_gem_path = File.join cache_dir, gem_file_name
 
     require "fileutils"
@@ -131,30 +147,26 @@ class Gem::RemoteFetcher
       nil
     end unless File.exist? cache_dir
 
-    source_uri = Gem::Uri.new(source_uri)
-
-    scheme = source_uri.scheme
-
-    # Gem::URI.parse gets confused by MS Windows paths with forward slashes.
-    scheme = nil if /^[a-z]$/i.match?(scheme)
-
     # REFACTOR: split this up and dispatch on scheme (eg download_http)
     # REFACTOR: be sure to clean up fake fetcher when you do this... cleaner
     case scheme
-    when "http", "https", "s3" then
-      unless File.exist? local_gem_path
+    when *REMOTE_SCHEMES then
+      if File.exist? local_gem_path
+        verbose "Using local gem #{local_gem_path}"
+      else
         begin
-          verbose "Downloading gem #{gem_file_name}"
+          verbose "Downloading gem #{gem_file_name} to #{cache_dir}"
 
           remote_gem_path = source_uri + "gems/#{gem_file_name}"
 
           cache_update_path remote_gem_path, local_gem_path
         rescue FetchError
-          raise if spec.original_platform == spec.platform
+          raise if Gem::ContentAddress.content_addressed?(spec, validate_ruby_abi: false)
+          raise if spec.original_platform.to_s == spec.platform.to_s
 
           alternate_name = "#{spec.original_name}.gem"
 
-          verbose "Failed, downloading gem #{alternate_name}"
+          verbose "Failed, downloading gem #{alternate_name} to #{cache_dir}"
 
           remote_gem_path = source_uri + "gems/#{alternate_name}"
 
@@ -168,7 +180,7 @@ class Gem::RemoteFetcher
 
         remote_gem_path = Gem::Util.correct_for_windows_path(File.join(path, "gems", gem_file_name))
 
-        FileUtils.cp(remote_gem_path, local_gem_path)
+        atomic_copy(remote_gem_path, local_gem_path)
       rescue Errno::EACCES
         local_gem_path = source_uri.to_s
       end
@@ -185,7 +197,7 @@ class Gem::RemoteFetcher
       source_path = Gem::UriFormatter.new(source_path).unescape
 
       begin
-        FileUtils.cp source_path, local_gem_path unless
+        atomic_copy(source_path, local_gem_path) unless
           File.identical?(source_path, local_gem_path)
       rescue Errno::EACCES
         local_gem_path = source_uri.to_s
@@ -234,7 +246,9 @@ class Gem::RemoteFetcher
 
       fetch_http(location, last_modified, head, depth + 1)
     else
-      raise FetchError.new("bad response #{response.message} #{response.code}", uri)
+      custom_error = response["X-Error-Message"]
+      error_detail = custom_error || response.message
+      raise FetchError.new("Bad response #{error_detail} #{response.code}", uri)
     end
   end
 
@@ -333,6 +347,40 @@ class Gem::RemoteFetcher
 
   private
 
+  # Creates +cache_dir+ so its writability can be probed, since File.writable?
+  # is false for a path that does not exist yet.
+
+  def ensure_writable_cache_dir(cache_dir)
+    require "fileutils"
+    begin
+      FileUtils.mkdir_p cache_dir
+    rescue SystemCallError
+      return false
+    end
+
+    File.writable?(cache_dir)
+  end
+
+  def atomic_copy(source_path, destination_path)
+    File.open(source_path, "rb") do |source|
+      # FileUtils.cp passed the source mode to File.open, so it only reached a
+      # file being created and the umask still applied to it. The writer
+      # already carries over the mode of a file it replaces.
+      mode = source.stat.mode & 0o777 & ~File.umask
+      replacing = File.exist?(destination_path)
+
+      Gem::AtomicFileWriter.open(destination_path) do |io|
+        IO.copy_stream(source, io)
+
+        begin
+          io.chmod(mode) unless replacing
+        rescue Errno::EPERM, Errno::EACCES
+          # the filesystem does not carry permissions
+        end
+      end
+    end
+  end
+
   def proxy_for(proxy, uri)
     Gem::Request.proxy_uri(proxy || Gem::Request.get_proxy_from_env(uri.scheme))
   end

data/lib/rubygems/request/connection_pools.rb

@@ -50,6 +50,9 @@ class Gem::Request::ConnectionPools # :nodoc:
   end
 
   def no_proxy?(host, env_no_proxy)
+    # A lone "*" entry bypasses the proxy for every host
+    return true if env_no_proxy.include?("*")
+
     host = host.downcase
 
     env_no_proxy.any? do |pattern|

data/lib/rubygems/request_set/gem_dependency_api.rb

@@ -13,7 +13,7 @@
 #   source 'https://rubygems.org'
 #
 #   gem 'rails', '3.2.14a
-#   gem 'devise', '~> 2.1', '>= 2.1.3'
+#   gem 'devise', '>= 2.1.3', '< 3'
 #   gem 'cancan'
 #   gem 'airbrake'
 #   gem 'pg'

data/lib/rubygems/request_set/lockfile.rb

@@ -231,5 +231,3 @@ class Gem::RequestSet::Lockfile
     @set.sorted_requests
   end
 end
-
-require_relative "lockfile/tokenizer"

data/lib/rubygems/request_set.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
+require_relative "cooldown"
 require_relative "vendored_tsort"
 
 ##
 # A RequestSet groups a request to activate a set of dependencies.
 #
-#   nokogiri = Gem::Dependency.new 'nokogiri', '~> 1.6'
-#   pg = Gem::Dependency.new 'pg', '~> 0.14'
+#   nokogiri = Gem::Dependency.new 'nokogiri', '>= 1.6'
+#   pg = Gem::Dependency.new 'pg', '>= 0.14'
 #
 #   set = Gem::RequestSet.new nokogiri, pg
 #
@@ -23,6 +24,11 @@ class Gem::RequestSet
 
   attr_accessor :always_install
 
+  ##
+  # The Gem::Cooldown applied to release candidates, if any.
+
+  attr_accessor :cooldown
+
   attr_reader :dependencies
 
   attr_accessor :development
@@ -86,8 +92,8 @@ class Gem::RequestSet
   # Creates a RequestSet for a list of Gem::Dependency objects, +deps+.  You
   # can then #resolve and #install the resolved list of dependencies.
   #
-  #   nokogiri = Gem::Dependency.new 'nokogiri', '~> 1.6'
-  #   pg = Gem::Dependency.new 'pg', '~> 0.14'
+  #   nokogiri = Gem::Dependency.new 'nokogiri', '>= 1.6'
+  #   pg = Gem::Dependency.new 'pg', '>= 0.14'
   #
   #   set = Gem::RequestSet.new nokogiri, pg
 
@@ -96,6 +102,7 @@ class Gem::RequestSet
 
     @always_install      = []
     @conservative        = false
+    @cooldown            = nil
     @dependency_names    = {}
     @development         = false
     @development_shallow = false
@@ -182,7 +189,7 @@ class Gem::RequestSet
     # Install requested gems after they have been downloaded
     sorted_requests.each do |req|
       if req.installed? && @always_install.none? {|spec| spec == req.spec.spec }
-        req.spec.spec.build_extensions
+        req.spec.spec.build_extensions unless options[:build_extension] == false
         yield req, nil if block_given?
         next
       end
@@ -223,6 +230,7 @@ class Gem::RequestSet
     @prerelease  = options[:prerelease]
     @remote      = options[:domain] != :local
     @conservative = true if options[:conservative]
+    @cooldown = Gem::Cooldown.from_options options
 
     gem_deps_api = load_gemdeps gemdeps, options[:without_groups], true
 
@@ -234,10 +242,6 @@ class Gem::RequestSet
       sorted_requests.each do |spec|
         puts "  #{spec.full_name}"
       end
-
-      if Gem.configuration.really_verbose
-        @resolver.stats.display
-      end
     else
       installed = install options, &block
 
@@ -322,11 +326,8 @@ class Gem::RequestSet
     @git_set.root_dir = @install_dir
 
     lock_file = "#{File.expand_path(path)}.lock"
-    begin
-      tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file lock_file
-      parser = tokenizer.make_parser self, []
-      parser.parse
-    rescue Errno::ENOENT
+    if File.exist?(lock_file)
+      load_lockfile lock_file
     end
 
     gf = Gem::RequestSet::GemDependencyAPI.new self, path
@@ -335,6 +336,91 @@ class Gem::RequestSet
     gf.load
   end
 
+  def load_lockfile(lock_file) # :nodoc:
+    require "bundler"
+    require "bundler/lockfile_parser"
+
+    # Bundler::Source::Path resolves relative `remote:` paths against
+    # Bundler.root, which raises when there is no Gemfile in the working
+    # directory. Anchor it to the lockfile's directory so PATH sections in a
+    # `gem install -g` lockfile can be parsed without a Bundler environment.
+    previous_root = Bundler.instance_variable_get(:@root)
+    Bundler.instance_variable_set(:@root, Pathname.new(File.expand_path(File.dirname(lock_file))))
+    root_swapped = true
+
+    # A PLUGIN SOURCE section otherwise sends Bundler::Plugin.from_lock looking
+    # for the plugin that handles it, which loads and runs that plugin's
+    # `plugins.rb`. Nothing here can use a plugin source anyway, so borrow the
+    # flag Bundler itself uses to keep lockfile parsing inert.
+    previous_gemfile_parse = Bundler::Plugin.instance_variable_get(:@gemfile_parse)
+    Bundler::Plugin.instance_variable_set(:@gemfile_parse, true)
+    gemfile_parse_swapped = true
+
+    parser = Bundler::LockfileParser.new(File.read(lock_file), lockfile_path: lock_file)
+
+    locked_versions = {}
+
+    parser.specs.group_by(&:source).each do |source, specs|
+      case source
+      when Bundler::Source::Rubygems
+        # Bundler::Source::Rubygems stores remotes in reverse of the lockfile
+        # order (Bundler::Source::Rubygems#to_lock reverses them back), so
+        # restore the lockfile order here.
+        remotes = source.remotes.reverse.map {|remote| Gem::Source.new(remote.to_s) }
+        remotes << Gem::Source.new(Gem::DEFAULT_HOST) if remotes.empty?
+        lock_set = Gem::Resolver::LockSet.new(remotes)
+        specs.each do |spec|
+          added = lock_set.add(spec.name, spec.version.to_s, spec.platform)
+          locked_versions[spec.name] ||= spec.version
+          spec.dependencies.each do |dep|
+            added.each {|s| s.add_dependency dep }
+          end
+        end
+        @sets << lock_set
+      when Bundler::Source::Git
+        git_set = Gem::Resolver::GitSet.new
+        git_set.root_dir = @install_dir
+        specs.each do |spec|
+          git_spec = git_set.add_git_spec(
+            spec.name,
+            spec.version.to_s,
+            source.uri.to_s,
+            source.revision,
+            source.submodules || false
+          )
+          locked_versions[spec.name] ||= spec.version
+          spec.dependencies.each {|dep| git_spec.add_dependency dep }
+        end
+        @sets << git_set
+      when Bundler::Source::Path
+        vendor_set = Gem::Resolver::VendorSet.new
+        specs.each do |spec|
+          loaded = vendor_set.add_vendor_gem(spec.name, source.path.to_s)
+          locked_versions[spec.name] ||= loaded.version
+          spec.dependencies.each {|dep| loaded.dependencies << dep }
+        end
+        @sets << vendor_set
+      end
+    end
+
+    parser.dependencies.each_value do |dep|
+      requirements = dep.requirement.as_list
+
+      # A dependency the lockfile ties to a source replaces whatever it asks
+      # for with the version that source resolved, the way the parser this
+      # replaced did. For a PATH section that is the version of the gemspec on
+      # disk, not the one the lockfile records.
+      if dep.source && (version = locked_versions[dep.name])
+        requirements = [version]
+      end
+
+      gem dep.name, *requirements
+    end
+  ensure
+    Bundler.instance_variable_set(:@root, previous_root) if root_swapped
+    Bundler::Plugin.instance_variable_set(:@gemfile_parse, previous_gemfile_parse) if gemfile_parse_swapped
+  end
+
   def pretty_print(q) # :nodoc:
     q.group 2, "[RequestSet:", "]" do
       q.breakable
@@ -392,6 +478,7 @@ class Gem::RequestSet
     set.prerelease = @prerelease
 
     resolver = Gem::Resolver.new @dependencies, set
+    resolver.cooldown            = @cooldown
     resolver.development         = @development
     resolver.development_shallow = @development_shallow
     resolver.ignore_dependencies = @ignore_dependencies
@@ -431,8 +518,10 @@ class Gem::RequestSet
   end
 
   def specs_in(dir)
-    Gem::Util.glob_files_in_dir("*.gemspec", File.join(dir, "specifications")).map do |g|
-      Gem::Specification.load g
+    Gem::SpecificationRecord.dirs_from([dir]).flat_map do |spec_dir|
+      Gem::Util.glob_files_in_dir("*.gemspec", spec_dir).map do |g|
+        Gem::Specification.load g
+      end
     end
   end
 
@@ -462,4 +551,3 @@ end
 
 require_relative "request_set/gem_dependency_api"
 require_relative "request_set/lockfile"
-require_relative "request_set/lockfile/tokenizer"

data/lib/rubygems/resolver/api_set.rb

@@ -35,6 +35,7 @@ class Gem::Resolver::APISet < Gem::Resolver::Set
     @dep_uri = dep_uri
     @uri     = dep_uri + ".."
 
+    @client = nil
     @data   = Hash.new {|h,k| h[k] = [] }
     @source = Gem::Source.new @uri
 
@@ -99,26 +100,20 @@ class Gem::Resolver::APISet < Gem::Resolver::Set
   # Return data for all versions of the gem +name+.
 
   def versions(name) # :nodoc:
-    if @data.key?(name)
-      return @data[name]
-    end
-
-    uri = @dep_uri + name
+    return @data[name] if @data.key?(name)
 
-    begin
-      str = Gem::RemoteFetcher.fetcher.fetch_path uri
-    rescue Gem::RemoteFetcher::FetchError
-      @data[name] = []
-    else
-      lines(str).each do |ver|
-        number, platform, dependencies, requirements = parse_gem(ver)
+    infos = begin
+      client.fetch_info(name)
+    rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error
+      []
+    end
 
-        platform ||= "ruby"
-        dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] }
-        requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h
+    infos.each do |_, number, suffix, dependencies, requirements|
+      suffix ||= "ruby"
+      dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] }
+      requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h
 
-        @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements }
-      end
+      @data[name] << { name: name, number: number, suffix: suffix, dependencies: dependencies, requirements: requirements }
     end
 
     @data[name]
@@ -126,14 +121,7 @@ class Gem::Resolver::APISet < Gem::Resolver::Set
 
   private
 
-  def lines(str)
-    lines = str.split("\n")
-    header = lines.index("---")
-    header ? lines[header + 1..-1] : lines
-  end
-
-  def parse_gem(string)
-    @gem_parser ||= GemParser.new
-    @gem_parser.parse(string)
+  def client # :nodoc:
+    @client ||= @source.compact_index_client
   end
 end

data/lib/rubygems/resolver/api_specification.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../cooldown"
+
 ##
 # Represents a specification retrieved via the Compact Index API.
 #
@@ -31,13 +33,13 @@ class Gem::Resolver::APISpecification < Gem::Resolver::Specification
     @set = set
     @name = api_data[:name]
     @version = Gem::Version.new(api_data[:number]).freeze
-    @platform = Gem::Platform.new(api_data[:platform]).freeze
-    @original_platform = api_data[:platform].freeze
     @dependencies = api_data[:dependencies].map do |name, ver|
       Gem::Dependency.new(name, ver.split(/\s*,\s*/)).freeze
     end.freeze
     @required_ruby_version = Gem::Requirement.new(api_data.dig(:requirements, :ruby)).freeze
     @required_rubygems_version = Gem::Requirement.new(api_data.dig(:requirements, :rubygems)).freeze
+    @created_at = parse_created_at(api_data.dig(:requirements, :created_at))&.freeze
+    assign_platform(api_data)
   end
 
   def ==(other) # :nodoc:
@@ -45,15 +47,17 @@ class Gem::Resolver::APISpecification < Gem::Resolver::Specification
       @set          == other.set &&
       @name         == other.name &&
       @version      == other.version &&
-      @platform     == other.platform
+      @platform     == other.platform &&
+      @content_address == other.content_address
   end
 
   def hash
-    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash
+    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash ^ @content_address.hash
   end
 
   def fetch_development_dependencies # :nodoc:
-    spec = source.fetch_spec Gem::NameTuple.new @name, @version, @platform
+    suffix = @content_address || @platform
+    spec = source.fetch_spec Gem::NameTuple.new @name, @version, suffix
 
     @dependencies = spec.dependencies
   end
@@ -84,22 +88,59 @@ class Gem::Resolver::APISpecification < Gem::Resolver::Specification
   end
 
   ##
-  # Fetches a Gem::Specification for this APISpecification.
+  # A Gem::Specification stub built from the Compact Index data for this
+  # specification. The compact index carries everything needed to
+  # download and install the gem, so the Marshal gemspec is not fetched.
+  # Development dependencies are not included; see
+  # #fetch_development_dependencies.
 
   def spec # :nodoc:
-    @spec ||=
-      begin
-        tuple = Gem::NameTuple.new @name, @version, @platform
-        source.fetch_spec tuple
-      rescue Gem::RemoteFetcher::FetchError
-        raise if @original_platform == @platform
-
-        tuple = Gem::NameTuple.new @name, @version, @original_platform
-        source.fetch_spec tuple
+    @spec ||= Gem::Specification.new do |s|
+      s.name     = @name
+      s.version  = @version
+      s.platform = @platform
+      s.original_platform = @original_platform
+      s.required_ruby_version = @required_ruby_version
+      s.required_rubygems_version = @required_rubygems_version
+      s.content_address = @content_address
+
+      @dependencies.each do |dependency|
+        s.add_runtime_dependency dependency.name, *dependency.requirement.as_list
       end
+    end
   end
 
   def source # :nodoc:
     @set.source
   end
+
+  private
+
+  def assign_platform(api_data)
+    suffix = api_data[:suffix]
+    required_platform = required_platform_from(api_data.dig(:requirements, :platform))
+
+    if Gem::ContentAddress.content_addressed_row?(suffix, required_platform, @required_ruby_version)
+      @content_address = suffix.freeze
+      @platform = required_platform.freeze
+      @original_platform = required_platform.to_s.freeze
+    else
+      @content_address = nil
+      @platform = Gem::Platform.new(suffix).freeze
+      @original_platform = suffix.freeze
+    end
+  end
+
+  def required_platform_from(requirement)
+    platform = Array(requirement).last
+    return if platform.nil? || platform.empty?
+
+    Gem::Platform.new(platform)
+  end
+
+  def parse_created_at(value)
+    value = value.first if value.is_a?(Array)
+
+    Gem::Cooldown.parse_created_at(value)
+  end
 end

data/lib/rubygems/resolver/incompatibility.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class Gem::Resolver::Incompatibility < Gem::PubGrub::Incompatibility
+  attr_reader :extended_explanation
+
+  def initialize(terms, cause:, custom_explanation: nil, extended_explanation: nil)
+    @extended_explanation = extended_explanation
+    super(terms, cause: cause, custom_explanation: custom_explanation)
+  end
+end

data/lib/rubygems/resolver/index_specification.rb

@@ -15,7 +15,7 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification
   # The +name+, +version+ and +platform+ are the name, version and platform of
   # the gem.
 
-  def initialize(set, name, version, source, platform)
+  def initialize(set, name, version, source, platform, content_address: nil)
     super()
 
     @set = set
@@ -24,6 +24,7 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification
     @source = source
     @platform = Gem::Platform.new(platform.to_s)
     @original_platform = platform.to_s
+    @content_address = content_address
 
     @spec = nil
   end
@@ -60,11 +61,12 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification
     self.class === other &&
       @name == other.name &&
       @version == other.version &&
-      @platform == other.platform
+      @platform == other.platform &&
+      @content_address == other.content_address
   end
 
   def hash
-    @name.hash ^ @version.hash ^ @platform.hash
+    [@name, @version, @platform, @content_address].hash
   end
 
   def inspect # :nodoc:
@@ -93,7 +95,7 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification
   def spec # :nodoc:
     @spec ||=
       begin
-        tuple = Gem::NameTuple.new @name, @version, @original_platform
+        tuple = Gem::NameTuple.new @name, @version, @original_platform, content_address: @content_address
 
         @source.fetch_spec tuple
       end

data/lib/rubygems/resolver/installer_set.rb

@@ -160,10 +160,10 @@ class Gem::Resolver::InstallerSet < Gem::Resolver::Set
       res.concat matching_local
 
       begin
-        if local_spec = @local_source.find_gem(name, dep.requirement)
+        @local_source.find_all_gems(name, dep.requirement).each do |local_spec|
           res << Gem::Resolver::IndexSpecification.new(
             self, local_spec.name, local_spec.version,
-            @local_source, local_spec.platform
+            @local_source, local_spec.platform, content_address: local_spec.content_address
           )
         end
       rescue Gem::Package::FormatError

data/lib/rubygems/resolver/lock_specification.rb

@@ -30,7 +30,11 @@ class Gem::Resolver::LockSpecification < Gem::Resolver::Specification
   def install(options = {})
     destination = options[:install_dir] || Gem.dir
 
-    if File.exist? File.join(destination, "specifications", spec.spec_name)
+    installed = Gem::SpecificationRecord.dirs_from([destination]).any? do |spec_dir|
+      File.exist? File.join(spec_dir, spec.spec_name)
+    end
+
+    if installed
       yield nil
       return
     end

data/lib/rubygems/resolver/spec_specification.rb

@@ -23,6 +23,10 @@ class Gem::Resolver::SpecSpecification < Gem::Resolver::Specification
     spec.dependencies
   end
 
+  def content_address # :nodoc:
+    spec.content_address
+  end
+
   ##
   # The required_ruby_version constraint for this specification