rubygems-update 4.0.21 → 4.1.0.beta1

data/lib/rubygems/source.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "text"
+require_relative "cooldown"
 ##
 # A Source knows how to list and fetch gems from a RubyGems marshal index.
 #
@@ -17,6 +18,24 @@ class Gem::Source
     prerelease: "prerelease_specs",
   }.freeze
 
+  ##
+  # Decoded compact index metadata for one content-addressable gem build:
+  # its +version+, content-address +suffix+, +ruby_abi+, and +platform+.
+  # Two infos are equal when their version and suffix match.
+
+  ContentAddressableInfo = Struct.new(:version, :suffix, :ruby_abi, :platform) do
+    def hash
+      [version, suffix].hash
+    end
+
+    def eql?(other)
+      other.is_a?(self.class) &&
+        version == other.version &&
+        suffix == other.suffix
+    end
+    alias_method :==, :eql?
+  end
+
   ##
   # The URI this source will fetch gems from.
 
@@ -162,8 +181,134 @@ class Gem::Source
   # :latest     => Return the list of only the highest version of each gem
   # :prerelease => Return the list of all prerelease only specs
   #
+  # The compact index is used when the source provides it, falling back
+  # to the Marshal spec indexes.
 
   def load_specs(type)
+    load_compact_index_specs(type) || load_marshal_specs(type)
+  end
+
+  ##
+  # The compact index client for this source, caching under
+  # Gem.spec_cache_dir. Also used by Gem::Resolver::APISet.
+
+  def compact_index_client # :nodoc:
+    @compact_index_client ||= begin
+      require_relative "compact_index_client"
+
+      index_uri = compact_index_uri
+
+      Gem::CompactIndexClient.new(compact_index_cache_dir(index_uri),
+        Gem::CompactIndexClient::HTTPFetcher.new(index_uri))
+    end
+  end
+
+  ##
+  # Decodes content-addressable +tuples+ (whose platform field holds the
+  # content address) into tuples carrying their real platform and Ruby ABI
+  # from the compact index /info metadata.
+
+  def decode_content_addressable_tuples(tuples, latest: false)
+    ca_tuples = tuples.select(&:content_address)
+    return tuples if ca_tuples.empty?
+
+    decoded_tuples = ca_tuples.group_by(&:name).flat_map do |name, name_tuples|
+      rows = name_tuples.map do |tuple|
+        [tuple.name, tuple.version, tuple.version.to_s, tuple.content_address]
+      end
+
+      content_addressable_tuples(name, rows)
+    end
+
+    decoded_by_key = decoded_tuples.to_h do |tuple|
+      [[tuple.name, tuple.version, tuple.content_address], tuple]
+    end
+
+    decoded = tuples.filter_map do |tuple|
+      if tuple.content_address
+        decoded_by_key[[tuple.name, tuple.version, tuple.content_address]]
+      else
+        tuple
+      end
+    end
+
+    return decoded unless latest
+
+    decoded.group_by(&:name).flat_map do |_, name_tuples|
+      max_versions_by_platform(name_tuples)
+    end
+  end
+
+  ##
+  # The publish time of gem +name+ at +version+ for +suffix+, when this
+  # source provides it through the compact index created_at metadata.
+  # Returns nil when the source, the gem or the version has no known
+  # publish time.
+
+  def created_at(name, version, suffix = Gem::Platform::RUBY)
+    return unless %w[http https].include?(uri.scheme)
+
+    @created_at_info ||= {}
+    info = @created_at_info[name] ||= begin
+      compact_index_client.fetch_info(name)
+    rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error
+      []
+    end
+
+    suffix = (suffix || Gem::Platform::RUBY).to_s
+    version = version.to_s
+
+    row = info.find do |row_info|
+      row_info[Gem::CompactIndexClient::INFO_VERSION] == version &&
+        (row_info[Gem::CompactIndexClient::INFO_SUFFIX] || Gem::Platform::RUBY) == suffix
+    end
+    return unless row
+
+    value = row[Gem::CompactIndexClient::INFO_REQS].assoc("created_at")&.last&.first
+
+    Gem::Cooldown.parse_created_at(value)
+  end
+
+  ##
+  # The publish time for +tuple+. Content-addressable tuples are looked up by
+  # content address; all other tuples are looked up by platform.
+
+  def created_at_for_tuple(tuple)
+    created_at(tuple.name, tuple.version, tuple.content_address || tuple.platform)
+  end
+
+  ##
+  # Downloads +spec+ and writes it to +dir+.  See also
+  # Gem::RemoteFetcher#download.
+
+  def download(spec, dir = Dir.pwd)
+    fetcher = Gem::RemoteFetcher.fetcher
+    fetcher.download spec, uri.to_s, dir
+  end
+
+  def pretty_print(q) # :nodoc:
+    q.object_group(self) do
+      q.group 2, "[Remote:", "]" do
+        q.breakable
+        q.text @uri.to_s
+
+        if api = uri
+          q.breakable
+          q.text "API URI: "
+          q.text api.to_s
+        end
+      end
+    end
+  end
+
+  def typo_squatting?(host, distance_threshold = 4)
+    return if @uri.host.nil?
+    levenshtein_distance(@uri.host, host).between? 1, distance_threshold
+  end
+
+  private
+
+  def load_marshal_specs(type)
     file       = FILES[type]
     fetcher    = Gem::RemoteFetcher.fetcher
     file_name  = "#{file}.#{Gem.marshal_version}"
@@ -194,48 +339,150 @@ class Gem::Source
   end
 
   ##
-  # Downloads +spec+ and writes it to +dir+.  See also
-  # Gem::RemoteFetcher#download.
+  # Builds the name tuple list for +type+ from the compact index versions
+  # file. Returns nil when the source does not provide a usable compact
+  # index, so the caller can fall back to the Marshal spec indexes.
 
-  def download(spec, dir = Dir.pwd)
-    fetcher = Gem::RemoteFetcher.fetcher
-    fetcher.download spec, uri.to_s, dir
+  def load_compact_index_specs(type)
+    return unless %w[http https].include?(uri.scheme)
+
+    versions = compact_index_versions
+    return if versions.nil? || versions.empty?
+
+    tuples = []
+
+    versions.each_value do |rows|
+      gem_tuples = rows.filter_map do |row_name, version_string, suffix|
+        next unless Gem::Version.correct?(version_string)
+
+        version = Gem::Version.new(version_string)
+        next if version.prerelease? != (type == :prerelease)
+
+        suffix ||= "ruby"
+        content_address = suffix if Gem::ContentAddress.match?(suffix)
+
+        Gem::NameTuple.new(row_name, version, suffix, content_address: content_address)
+      end
+
+      gem_tuples = max_versions_by_platform(gem_tuples) if type == :latest
+
+      tuples.concat(gem_tuples)
+    end
+
+    tuples
   end
 
-  def pretty_print(q) # :nodoc:
-    q.object_group(self) do
-      q.group 2, "[Remote:", "]" do
-        q.breakable
-        q.text @uri.to_s
+  def compact_index_versions
+    @compact_index_versions ||= compact_index_client.versions
+  rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error
+    @compact_index_versions = {}
+    nil
+  end
 
-        if api = uri
-          q.breakable
-          q.text "API URI: "
-          q.text api.to_s
-        end
+  def compact_index_info_rows(name)
+    compact_index_client.info(name)
+  rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error
+    []
+  end
+
+  def content_addressable_tuples(name, rows)
+    metadata = content_addressable_metadata(name, rows)
+
+    rows.filter_map do |row_name, version, version_string, suffix|
+      row_metadata = metadata.find do |entry|
+        entry.version == version_string && entry.suffix == suffix
       end
+      next unless row_metadata
+
+      Gem::NameTuple.new(
+        row_name,
+        version,
+        row_metadata.platform,
+        content_address: suffix,
+        ruby_abi: row_metadata.ruby_abi
+      )
     end
   end
 
-  def typo_squatting?(host, distance_threshold = 4)
-    return if @uri.host.nil?
-    levenshtein_distance(@uri.host, host).between? 1, distance_threshold
+  def content_addressable_metadata(name, rows)
+    wanted_rows = rows.map do |row|
+      ContentAddressableInfo.new(row[2], row[3], nil, nil)
+    end
+
+    available_rows = compact_index_info_rows(name).filter_map do |info_row|
+      version = info_row[Gem::CompactIndexClient::INFO_VERSION]
+      suffix = info_row[Gem::CompactIndexClient::INFO_SUFFIX]
+
+      requirements = compact_index_requirements(info_row)
+      platform = required_platform_from(requirements[:platform])
+      next unless platform
+      next unless requirements[:ruby]
+
+      ruby_abi = Gem::ContentAddress.ruby_abi_for(Gem::Requirement.new(requirements[:ruby]))
+      ContentAddressableInfo.new(version, suffix, ruby_abi, platform)
+    end
+
+    available_rows & wanted_rows
   end
 
-  private
+  def compact_index_requirements(info_row)
+    info_row[Gem::CompactIndexClient::INFO_REQS].to_h do |key, requirements|
+      [key.to_sym, requirements]
+    end
+  end
 
-  def new_dependency_resolver_set
-    return Gem::Resolver::IndexSet.new self if uri.scheme == "file"
+  def required_platform_from(requirement)
+    platform = Array(requirement).last.to_s
+    return if platform.empty?
+
+    platform
+  end
 
-    fetch_uri = if uri.host == "rubygems.org"
+  def max_versions_by_platform(tuples)
+    grouped_tuples = tuples.group_by {|tuple| latest_platform_key(tuple) }
+    grouped_tuples.map do |_, platform_tuples|
+      platform_tuples.max_by(&:version)
+    end
+  end
+
+  def latest_platform_key(tuple)
+    if tuple.content_address
+      [tuple.platform, tuple.ruby_abi || tuple.content_address]
+    else
+      tuple.platform
+    end
+  end
+
+  def compact_index_uri
+    if uri.host == "rubygems.org"
       index_uri = uri.dup
       index_uri.host = "index.rubygems.org"
       index_uri
     else
       uri
     end
+  end
+
+  def compact_index_cache_dir(index_uri)
+    if update_cache?
+      # Correct for windows paths
+      escaped_path = index_uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/')
+
+      File.join Gem.spec_cache_dir, "compact_index",
+        "#{index_uri.host}%#{index_uri.port}", *escaped_path.split("/").reject(&:empty?)
+    else
+      require "tmpdir"
+      require "fileutils"
+      dir = Dir.mktmpdir "gem_compact_index"
+      at_exit { FileUtils.rm_rf dir }
+      dir
+    end
+  end
+
+  def new_dependency_resolver_set
+    return Gem::Resolver::IndexSet.new self if uri.scheme == "file"
 
-    bundler_api_uri = enforce_trailing_slash(fetch_uri) + "versions"
+    bundler_api_uri = enforce_trailing_slash(compact_index_uri) + "versions"
 
     begin
       fetcher = Gem::RemoteFetcher.fetcher

data/lib/rubygems/spec_fetcher.rb

@@ -82,13 +82,17 @@ class Gem::SpecFetcher
   # Find and fetch gem name tuples that match +dependency+.
   #
   # If +matching_platform+ is false, gems for all platforms are returned.
+  #
+  # +type+ overrides the index type derived from +dependency+.  See
+  # #available_specs for the list of types.
 
-  def search_for_dependency(dependency, matching_platform = true)
+  def search_for_dependency(dependency, matching_platform = true, type: nil)
     found = {}
 
     rejected_specs = {}
 
-    list, errors = available_specs(dependency.identity)
+    specs_type = type || dependency.identity
+    list, errors = available_specs(specs_type)
 
     list.each do |source, specs|
       if dependency.name.is_a?(String) && specs.respond_to?(:bsearch)
@@ -97,17 +101,20 @@ class Gem::SpecFetcher
         specs = specs[start_index...end_index] if start_index && end_index
       end
 
+      specs = specs.select {|tup| dependency.match?(tup) }
+      specs = decode_source_content_addressable_tuples(source, specs, latest: specs_type == :latest)
+
       found[source] = specs.select do |tup|
-        if dependency.match?(tup)
-          if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name)
-            pm = (
-              rejected_specs[dependency] ||= \
-                Gem::PlatformMismatch.new(tup.name, tup.version))
-            pm.add_platform tup.platform
-            false
-          else
-            true
-          end
+        if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name)
+          pm = (
+            rejected_specs[dependency] ||= \
+              Gem::PlatformMismatch.new(tup.name, tup.version))
+          pm.add_platform tup.platform
+          false
+        elsif matching_platform && !ruby_abi_match?(tup)
+          false
+        else
+          true
         end
       end
     end
@@ -156,6 +163,7 @@ class Gem::SpecFetcher
     specs = []
     tuples.each do |tup, source|
       spec = source.fetch_spec(tup)
+      spec.content_address = tup.content_address if tup.content_address
     rescue Gem::RemoteFetcher::FetchError => e
       errors << Gem::SourceFetchProblem.new(source, e)
     else
@@ -165,6 +173,17 @@ class Gem::SpecFetcher
     [specs, errors]
   end
 
+  ##
+  # Decodes the content-addressable tuples in +spec_tuples+ ([tuple, source]
+  # pairs) to carry their real platform and Ruby ABI via each source.
+
+  def decode_content_addressable_tuples(spec_tuples, latest: false)
+    spec_tuples.group_by {|_, source| source }.flat_map do |source, source_tuples|
+      tuples = source_tuples.map(&:first)
+      decode_source_content_addressable_tuples(source, tuples, latest: latest).map {|tuple| [tuple, source] }
+    end
+  end
+
   ##
   # Suggests gems based on the supplied +gem_name+. Returns an array of
   # alternative gem names.
@@ -287,4 +306,20 @@ class Gem::SpecFetcher
     raise unless gracefully_ignore
     []
   end
+
+  private
+
+  def decode_source_content_addressable_tuples(source, tuples, latest: false) # :nodoc:
+    return tuples unless source.respond_to?(:decode_content_addressable_tuples)
+
+    source.decode_content_addressable_tuples(tuples, latest: latest)
+  end
+
+  def ruby_abi_match?(tuple) # :nodoc:
+    !tuple.ruby_abi || tuple.ruby_abi == current_ruby_abi
+  end
+
+  def current_ruby_abi # :nodoc:
+    @current_ruby_abi ||= Gem.ruby_abi
+  end
 end

data/lib/rubygems/specification.rb

@@ -11,7 +11,6 @@ require_relative "basic_specification"
 require_relative "stub_specification"
 require_relative "platform"
 require_relative "specification_record"
-require_relative "util/list"
 
 require "rbconfig"
 
@@ -420,6 +419,13 @@ class Gem::Specification < Gem::BasicSpecification
 
   attr_accessor :metadata
 
+  ##
+  # The content address of this gem, a SHA-256 prefix of the gem file
+  # contents used in place of the platform in file and install names
+  # (e.g. "example-1.0-78be552b"), or +nil+ for non-content-addressable gems.
+
+  attr_accessor :content_address
+
   ######################################################################
   # :section: Optional gemspec attributes
 
@@ -545,7 +551,7 @@ class Gem::Specification < Gem::BasicSpecification
   #
   # Usage:
   #
-  #   spec.add_development_dependency 'example', '~> 1.1', '>= 1.1.4'
+  #   spec.add_development_dependency 'example', '>= 1.1.4', '< 2'
   #
   # Development dependencies aren't installed by default and aren't
   # activated when a gem is required.
@@ -559,7 +565,7 @@ class Gem::Specification < Gem::BasicSpecification
   #
   # Usage:
   #
-  #   spec.add_dependency 'example', '~> 1.1', '>= 1.1.4'
+  #   spec.add_dependency 'example', '>= 1.1.4', '< 2'
 
   def add_dependency(gem, *requirements)
     if requirements.uniq.size != requirements.size
@@ -569,6 +575,15 @@ class Gem::Specification < Gem::BasicSpecification
     add_dependency_with_type(gem, :runtime, requirements)
   end
 
+  ##
+  # Ruby ABI of the gem derived from required_ruby_version
+  # Only supports required_ruby_version in "~> X.Y.0" format (single pessimistic requirement with 3 segments)
+  # Returns nil if the required_ruby_version does not specify a single Ruby ABI
+
+  def ruby_abi
+    Gem::ContentAddress.ruby_abi_for(required_ruby_version)
+  end
+
   ##
   # Executables included in the gem.
   #
@@ -1248,7 +1263,9 @@ class Gem::Specification < Gem::BasicSpecification
   # Keeps track of all currently known specifications
 
   def self.specification_record
-    @specification_record ||= Gem::SpecificationRecord.new(dirs)
+    @specification_record ||= Gem::SpecificationRecord.new(
+      Gem::SpecificationRecord.dirs_with_abi(dirs)
+    )
   end
 
   # DOC: This method needs documented or nodoc'd
@@ -1288,7 +1305,7 @@ class Gem::Specification < Gem::BasicSpecification
       raise unless message.include?("YAML::")
 
       unless Object.const_defined?(:YAML)
-        Object.const_set "YAML", Psych
+        Object.const_set "YAML", Module.new
         yaml_set = true
       end
 
@@ -1297,7 +1314,7 @@ class Gem::Specification < Gem::BasicSpecification
 
         YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") && !YAML::Syck.const_defined?(:DefaultKey)
       elsif message.include?("YAML::PrivateType") && !YAML.const_defined?(:PrivateType)
-        YAML.const_set "PrivateType", Class.new
+        YAML.const_set "PrivateType", Class.new { attr_accessor :type_id, :value }
       end
 
       retry_count += 1
@@ -1355,7 +1372,8 @@ class Gem::Specification < Gem::BasicSpecification
     self.class === other &&
       name == other.name &&
       version == other.version &&
-      platform == other.platform
+      platform == other.platform &&
+      content_address == other.content_address
   end
 
   ##
@@ -1926,7 +1944,7 @@ class Gem::Specification < Gem::BasicSpecification
   # :startdoc:
 
   def hash # :nodoc:
-    name.hash ^ version.hash
+    [name, version, platform, content_address].hash
   end
 
   def init_with(coder) # :nodoc:
@@ -1962,6 +1980,7 @@ class Gem::Specification < Gem::BasicSpecification
     @loaded_from = nil
     @original_platform = nil
     @installed_by_version = nil
+    @content_address = nil
 
     set_nil_attributes_to_nil
     set_not_nil_attributes_to_default_values
@@ -2009,13 +2028,19 @@ class Gem::Specification < Gem::BasicSpecification
 
   def base_dir
     return Gem.dir unless loaded_from
-    @base_dir ||= if default_gem?
+    @base_dir ||= if default_gem? || loaded_from_abi_scoped_spec_dir?
       File.dirname File.dirname File.dirname loaded_from
     else
       File.dirname File.dirname loaded_from
     end
   end
 
+  def loaded_from_abi_scoped_spec_dir?
+    !loaded_from.nil? &&
+      Gem::SpecificationRecord.abi_scoped_spec_dir?(File.dirname(loaded_from))
+  end
+  private :loaded_from_abi_scoped_spec_dir?
+
   def inspect # :nodoc:
     if $DEBUG
       super
@@ -2115,7 +2140,7 @@ class Gem::Specification < Gem::BasicSpecification
   # Return a NameTuple that represents this Specification
 
   def name_tuple
-    Gem::NameTuple.new name, version, original_platform
+    Gem::NameTuple.new name, version, original_platform, content_address: content_address
   end
 
   ##
@@ -2204,10 +2229,13 @@ class Gem::Specification < Gem::BasicSpecification
   end
 
   ##
-  # Sets rdoc_options to +value+, ensuring it is an array.
+  # Sets rdoc_options to +value+, ensuring it is a flat array of strings.
+  # Handles malformed gemspecs where rdoc_options may be a Hash or contain Hashes.
 
   def rdoc_options=(options)
-    @rdoc_options = Array options
+    @rdoc_options = Array(options).flat_map do |opt|
+      opt.is_a?(Hash) ? opt.to_a.flatten.map(&:to_s) : opt
+    end
   end
 
   ##
@@ -2279,7 +2307,8 @@ class Gem::Specification < Gem::BasicSpecification
   # True if this gem has the same attributes as +other+.
 
   def same_attributes?(spec)
-    @@attributes.all? {|name, _default| send(name) == spec.send(name) }
+    @@attributes.all? {|name, _default| send(name) == spec.send(name) } &&
+      content_address == spec.content_address
   end
 
   private :same_attributes?
@@ -2307,11 +2336,14 @@ class Gem::Specification < Gem::BasicSpecification
   end
 
   ##
-  # Returns the full path to the directory containing this spec's
-  # gemspec file. eg: /usr/local/lib/ruby/gems/1.8/specifications
-
+  # Full path to the directory containing this spec's gemspec file.
+  # ABI-scoped for content-addressed specs.
   def spec_dir
-    @spec_dir ||= File.join base_dir, "specifications"
+    @spec_dir ||= if loaded_from && Gem::ContentAddress.content_addressed?(self)
+      File.dirname loaded_from
+    else
+      Gem::SpecificationRecord.specification_dir_for(self, base_dir)
+    end
   end
 
   ##
@@ -2378,11 +2410,14 @@ class Gem::Specification < Gem::BasicSpecification
   # still have their default values are omitted.
 
   def to_ruby
+    content_addressed = Gem::ContentAddress.content_addressed?(self)
+    gem_suffix = content_addressed ? content_address : platform
     result = []
     result << "# -*- encoding: utf-8 -*-"
-    result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}"
+    result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{gem_suffix} #{raw_require_paths.join("\0")}"
     result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless
       extensions.empty?
+    result << "#{Gem::StubSpecification::TARGET_PREFIX}platform=#{platform}" if content_addressed
     result << nil
     result << "Gem::Specification.new do |s|"
 
@@ -2391,6 +2426,7 @@ class Gem::Specification < Gem::BasicSpecification
     unless platform.nil? || platform == Gem::Platform::RUBY
       result << "  s.platform = #{ruby_code original_platform}"
     end
+    result << "  s.content_address = #{ruby_code content_address} if s.respond_to? :content_address=" if content_addressed
     result << ""
     result << "  s.required_rubygems_version = #{ruby_code required_rubygems_version} if s.respond_to? :required_rubygems_version="
 
@@ -2469,24 +2505,28 @@ class Gem::Specification < Gem::BasicSpecification
   def to_yaml(opts = {}) # :nodoc:
     Gem.load_yaml
 
-    # Because the user can switch the YAML engine behind our
-    # back, we have to check again here to make sure that our
-    # psych code was properly loaded, and load it if not.
-    unless Gem.const_defined?(:NoAliasYAMLTree)
-      require_relative "psych_tree"
-    end
+    if Gem.use_psych?
+      # Because the user can switch the YAML engine behind our
+      # back, we have to check again here to make sure that our
+      # psych code was properly loaded, and load it if not.
+      unless Gem.const_defined?(:NoAliasYAMLTree)
+        require_relative "psych_tree"
+      end
 
-    builder = Gem::NoAliasYAMLTree.create
-    builder << self
-    ast = builder.tree
+      builder = Gem::NoAliasYAMLTree.create
+      builder << self
+      ast = builder.tree
 
-    require "stringio"
-    io = StringIO.new
-    io.set_encoding Encoding::UTF_8
+      require "stringio"
+      io = StringIO.new
+      io.set_encoding Encoding::UTF_8
 
-    Psych::Visitors::Emitter.new(io).accept(ast)
+      Psych::Visitors::Emitter.new(io).accept(ast)
 
-    io.string.gsub(/ !!null \n/, " \n")
+      io.string.gsub(/ !!null \n/, " \n")
+    else
+      Gem::YAMLSerializer.dump(self)
+    end
   end
 
   ##
@@ -2563,6 +2603,10 @@ class Gem::Specification < Gem::BasicSpecification
         self.date = val
       when "platform"
         self.platform = val
+      when "rdoc_options"
+        self.rdoc_options = val
+      when "requirements"
+        self.requirements = val
       else
         instance_variable_set "@#{ivar}", val
       end