rubygems-update 4.0.21 → 4.1.0.beta1

data/lib/rubygems/package.rb

@@ -129,13 +129,45 @@ class Gem::Package
   # Permission for other files
   attr_accessor :data_mode
 
-  def self.build(spec, skip_validation = false, strict_validation = false, file_name = nil)
-    gem_file = file_name || spec.file_name
+  ##
+  # The minimum RubyGems version that can install content-addressable gems.
+  # Built into +required_rubygems_version+ so older clients reject skinny
+  # gems through both the local and remote install paths.
+
+  MINIMUM_RUBYGEMS_VERSION = ">= 4.1.0.a"
+
+  ##
+  # Builds the gem described by +spec+ and returns the built file name;
+  # passing +ruby_abi+ ("X.Y") builds a content-addressable gem named by the
+  # SHA-256 of its contents, updates +spec.required_ruby_version+ to
+  # "~> X.Y.0", and constrains +spec.required_rubygems_version+ to at least
+  # MINIMUM_RUBYGEMS_VERSION (incompatible with
+  # +file_name+).
+
+  def self.build(spec, skip_validation = false, strict_validation = false, file_name = nil, ruby_abi = nil)
+    if ruby_abi && file_name
+      raise ArgumentError, "Cannot specify both a Ruby ABI and an output file name because content addressable gems must use the generated file name."
+    end
+    if ruby_abi
+      require "digest"
+      require "stringio"
+
+      io = StringIO.new
+      io.set_encoding(Encoding::BINARY)
 
-    package = new gem_file
-    package.spec = spec
-    package.build skip_validation, strict_validation
+      package = new io
+      package.spec = spec.dup
+      gem_file = package.build_content_addressable_file ruby_abi, skip_validation, strict_validation
 
+      spec.required_ruby_version = package.spec.required_ruby_version
+      spec.required_rubygems_version = package.spec.required_rubygems_version
+    else
+      gem_file = file_name || spec.file_name
+
+      package = new gem_file
+      package.spec = spec
+      package.build skip_validation, strict_validation
+    end
     gem_file
   end
 
@@ -216,6 +248,21 @@ class Gem::Package
     FileUtils.cp @gem.path, path unless File.exist? path
   end
 
+  ##
+  # Derives the content address from the gem's file name and verifies it
+  # against the SHA256 digest of the file contents.
+
+  def content_address
+    path = @gem&.path
+    return unless path
+
+    address = Gem::ContentAddress.verified_file_name_claim(path, spec)
+    return nil unless address
+    return nil unless Gem::ContentAddress.eligible?(spec)
+
+    address
+  end
+
   ##
   # Adds a checksum for each entry in the gem to checksums.yaml.gz.
 
@@ -232,7 +279,11 @@ class Gem::Package
 
     tar.add_file_signed "checksums.yaml.gz", 0o444, @signer do |io|
       gzip_to io do |gz_io|
-        Psych.dump checksums_by_algorithm, gz_io
+        if Gem.use_psych?
+          Psych.dump checksums_by_algorithm, gz_io
+        else
+          gz_io.write Gem::YAMLSerializer.dump(checksums_by_algorithm)
+        end
       end
     end
   end
@@ -311,16 +362,47 @@ class Gem::Package
       end
     end
 
-    say <<-EOM
+    message = <<-EOM
   Successfully built RubyGem
   Name: #{@spec.name}
   Version: #{@spec.version}
-  File: #{File.basename @gem.path}
 EOM
+
+    message += "  File: #{File.basename(@gem.path)}\n" if @gem.path
+    say message
   ensure
     @signer = nil
   end
 
+  ##
+  # Builds this package scoped to +ruby_abi+ ("X.Y"), then writes it to a
+  # content-addressable file name derived from the SHA-256 digest of the gem
+  # contents, e.g. "example-1.0-01234567.gem". Returns the file name of the
+  # written gem.
+  #
+  # The spec is validated for an ABI-scoped build and its
+  # +required_ruby_version+ and +required_rubygems_version+ are constrained
+  # before building, so every gem this method produces is eligible for
+  # content addressing.
+
+  def build_content_addressable_file(ruby_abi, skip_validation = false, strict_validation = false)
+    validate_ruby_abi ruby_abi
+    @spec.required_rubygems_version = normalized_required_rubygems_version(ruby_abi)
+    @spec.required_ruby_version = Gem::ContentAddress.ruby_abi_requirement(ruby_abi)
+
+    build skip_validation, strict_validation
+
+    bytes = @gem.with_read_io(&:read)
+    gem_file = "#{@spec.name}-#{@spec.version}-#{Gem::ContentAddress.address_for(bytes)}.gem"
+    File.binwrite(gem_file, bytes)
+
+    say "  File: #{gem_file}"
+    say "  Platform: #{@spec.platform}"
+    say "  Ruby ABI: #{ruby_abi}"
+
+    gem_file
+  end
+
   ##
   # A list of file names contained in this gem
 
@@ -474,7 +556,7 @@ EOM
       end
     end
 
-    if dir_mode
+    if dir_mode && !directories.empty?
       File.chmod(dir_mode, *directories)
     end
   end
@@ -503,24 +585,6 @@ EOM
     gz_io.close
   end
 
-  ##
-  # Returns the full path for installing +filename+.
-  #
-  # If +filename+ is not inside +destination_dir+ an exception is raised.
-
-  def install_location(filename, destination_dir) # :nodoc:
-    raise Gem::Package::PathError.new(filename, destination_dir) if
-      filename.start_with? "/"
-
-    destination_dir = File.realpath(destination_dir)
-    destination = File.expand_path(filename, destination_dir)
-
-    raise Gem::Package::PathError.new(destination, destination_dir) unless
-      normalize_path(destination).start_with? normalize_path(destination_dir + "/")
-
-    destination
-  end
-
   if Gem.win_platform?
     def normalize_path(pathname) # :nodoc:
       pathname.downcase
@@ -534,7 +598,7 @@ EOM
   ##
   # Loads a Gem::Specification from the TarEntry +entry+
 
-  def load_spec(entry) # :nodoc:
+  def load_spec_from_metadata(entry) # :nodoc:
     limit = 10 * 1024 * 1024
     case entry.full_name
     when "metadata" then
@@ -655,6 +719,110 @@ EOM
 
   private
 
+  ##
+  # The +required_rubygems_version+ for a content-addressable build: the
+  # spec's requirement raised to at least MINIMUM_RUBYGEMS_VERSION, warning
+  # if it had to be changed. Raises if the requirement excludes every version
+  # satisfying that floor, since no RubyGems could install the built gem.
+
+  def normalized_required_rubygems_version(ruby_abi)
+    minimum = Gem::Requirement.new(MINIMUM_RUBYGEMS_VERSION)
+    existing = @spec.required_rubygems_version
+
+    return minimum if existing.nil? || existing == Gem::Requirement.default
+
+    floor = minimum.requirements.first.last
+
+    if excludes_rubygems_floor?(existing, floor)
+      raise ArgumentError,
+        "Cannot build gem for Ruby ABI #{ruby_abi} because required_rubygems_version is set to #{existing}, " \
+        "which excludes RubyGems #{MINIMUM_RUBYGEMS_VERSION} required to install content addressable gems. " \
+        "Please remove or loosen the conflicting constraint."
+    end
+
+    return existing if satisfies_rubygems_floor?(existing, floor)
+
+    preserved = existing.requirements.filter_map do |op, version|
+      "#{op} #{version}" if ["~>", "<", "<=", "!="].include?(op)
+    end
+
+    normalized = Gem::Requirement.new([MINIMUM_RUBYGEMS_VERSION, *preserved])
+
+    alert_warning \
+      "required_rubygems_version was changed from \"#{existing}\" to \"#{normalized}\" for this build " \
+      "because content addressable gems can only be installed by RubyGems #{MINIMUM_RUBYGEMS_VERSION}."
+
+    normalized
+  end
+
+  ##
+  # Whether +requirement+ excludes every RubyGems version satisfying the
+  # +floor+, so that no RubyGems could install the built gem.
+
+  def excludes_rubygems_floor?(requirement, floor)
+    capped_below_floor = requirement.requirements.any? do |op, version|
+      case op
+      when "<" then version <= floor
+      when "<=", "=" then version < floor
+      when "~>" then version.bump <= floor.release
+      else false
+      end
+    end
+
+    return true if capped_below_floor
+
+    !requirement.satisfied_by?(floor) && requirement.requirements.any? do |op, version|
+      ["<=", "="].include?(op) && version == floor
+    end
+  end
+
+  ##
+  # Whether one of the lower bounds of +requirement+ already guarantees the
+  # +floor+.
+
+  def satisfies_rubygems_floor?(requirement, floor)
+    requirement.requirements.any? do |op, version|
+      case op
+      when ">=", "~>", "=", ">" then version >= floor
+      else false
+      end
+    end
+  end
+
+  ##
+  # Validates that the spec can be built as a content-addressable gem scoped
+  # to +ruby_abi+ ("X.Y"): the ABI must be well-formed, the spec must declare
+  # a non-Ruby platform, and any existing +required_ruby_version+ must match
+  # the ABI.
+
+  def validate_ruby_abi(ruby_abi)
+    if !Gem::ContentAddress.valid_ruby_abi?(ruby_abi)
+      raise ArgumentError, "Ruby ABI must be in X.Y format"
+    elsif !Gem::ContentAddress.platform_eligible?(@spec.platform)
+      raise ArgumentError, "Cannot build a gem scoped to a single Ruby ABI as no platform or a Ruby platform has been set"
+    elsif !Gem::ContentAddress.ruby_abi_compatible?(@spec, ruby_abi)
+      raise ArgumentError, "Cannot build gem for Ruby ABI #{ruby_abi} because required_ruby_version is set to #{@spec.required_ruby_version}. Please set required_ruby_version to \"~> #{ruby_abi}.0\"."
+    end
+  end
+
+  ##
+  # Returns the full path for installing +filename+ into +destination_dir+,
+  # which must already be resolved with File.realpath by the caller.
+  #
+  # If +filename+ is not inside +destination_dir+ an exception is raised.
+
+  def install_location(filename, destination_dir) # :nodoc:
+    raise Gem::Package::PathError.new(filename, destination_dir) if
+      filename.start_with? "/"
+
+    destination = File.expand_path(filename, destination_dir)
+
+    raise Gem::Package::PathError.new(destination, destination_dir) unless
+      normalize_path(destination).start_with? normalize_path(destination_dir + "/")
+
+    destination
+  end
+
   ##
   # Raises an exception unless +dir+, with symlinks resolved, is
   # +destination_dir+ or a directory inside it.  +destination_dir+ must
@@ -703,12 +871,7 @@ EOM
       digest entry
     end
 
-    case file_name
-    when "metadata", "metadata.gz" then
-      load_spec entry
-    when "data.tar.gz" then
-      verify_gz entry
-    end
+    load_spec_from_metadata entry
   rescue StandardError
     warn "Exception while verifying #{@gem.path}"
     raise
@@ -736,18 +899,6 @@ EOM
     end
   end
 
-  ##
-  # Verifies that +entry+ is a valid gzipped file.
-
-  def verify_gz(entry) # :nodoc:
-    Zlib::GzipReader.wrap entry do |gzio|
-      # TODO: read into a buffer once zlib supports it
-      gzio.read 16_384 until gzio.eof? # gzip checksum verification
-    end
-  rescue Zlib::GzipFile::Error => e
-    raise Gem::Package::FormatError.new(e.message, entry.full_name)
-  end
-
   if RUBY_ENGINE == "truffleruby"
     def copy_stream(src, dst, size) # :nodoc:
       dst.write src.read(size)
@@ -767,9 +918,11 @@ EOM
   if Gem.win_platform?
     # Create a symlink and fallback to copy the file or directory on Windows,
     # where symlink creation needs special privileges in form of the Developer Mode.
+    # JRuby on Windows raises TypeError from the wincode path-conversion helper
+    # when it cannot create the symlink, so fall back to copy in that case too.
     def create_symlink(old_name, new_name)
       File.symlink(old_name, new_name)
-    rescue Errno::EACCES
+    rescue Errno::EACCES, TypeError
       from = File.expand_path(old_name, File.dirname(new_name))
       FileUtils.cp_r(from, new_name)
     end

data/lib/rubygems/package_task.rb

@@ -111,10 +111,10 @@ class Gem::PackageTask < Rake::PackageTask
     file gem_path => [package_dir, gem_dir] + @gem_spec.files do
       chdir(gem_dir) do
         when_writing "Creating #{gem_spec.file_name}" do
-          Gem::Package.build gem_spec
+          built_gem_file = Gem::Package.build gem_spec
 
           verbose trace do
-            mv gem_file, ".."
+            mv built_gem_file, ".."
           end
         end
       end

data/lib/rubygems/platform.rb

@@ -196,7 +196,7 @@ class Gem::Platform
   # the runtime platform.
   #
   #--
-  # NOTE: Until it can be removed, changes to this method must also be reflected in `bundler/lib/bundler/rubygems_ext.rb`
+  # NOTE: Until it can be removed, changes to this method must also be reflected in `lib/bundler/rubygems_ext.rb`
 
   def ===(other)
     return nil unless Gem::Platform === other
@@ -221,7 +221,7 @@ class Gem::Platform
   end
 
   #--
-  # NOTE: Until it can be removed, changes to this method must also be reflected in `bundler/lib/bundler/rubygems_ext.rb`
+  # NOTE: Until it can be removed, changes to this method must also be reflected in `lib/bundler/rubygems_ext.rb`
 
   def normalized_linux_version
     return nil unless @version

data/lib/rubygems/query_utils.rb

@@ -149,9 +149,15 @@ module Gem::QueryUtils
     spec_tuples = if name.nil?
       fetcher.detect(specs_type) { true }
     else
-      fetcher.detect(specs_type) do |name_tuple|
+      matching_tuples = fetcher.detect(specs_type) do |name_tuple|
         name === name_tuple.name && options[:version].satisfied_by?(name_tuple.version)
       end
+
+      if args.empty?
+        matching_tuples
+      else
+        fetcher.decode_content_addressable_tuples(matching_tuples, latest: specs_type == :latest)
+      end
     end
 
     output_query_results(spec_tuples)
@@ -200,9 +206,11 @@ module Gem::QueryUtils
       matching_tuples = matching_tuples.sort_by {|n,_| n.version }.reverse
 
       platforms = Hash.new {|h,version| h[version] = [] }
+      platform_ruby_abis = Hash.new {|h,version| h[version] = Hash.new {|hh,platform| hh[platform] = [] } }
 
       matching_tuples.each do |n, _|
         platforms[n.version] << n.platform if n.platform
+        platform_ruby_abis[n.version][n.platform] << n.ruby_abi if n.ruby_abi
       end
 
       seen = {}
@@ -216,11 +224,11 @@ module Gem::QueryUtils
         end
       end
 
-      output << clean_text(make_entry(matching_tuples, platforms))
+      output << clean_text(make_entry(matching_tuples, platforms, platform_ruby_abis))
     end
   end
 
-  def entry_details(entry, detail_tuple, specs, platforms)
+  def entry_details(entry, detail_tuple, specs, platforms, platform_ruby_abis)
     return unless options[:details]
 
     name_tuple, spec = detail_tuple
@@ -229,7 +237,11 @@ module Gem::QueryUtils
 
     entry << "\n"
 
-    spec_platforms   entry, platforms
+    if ruby_abi_metadata?(platform_ruby_abis)
+      spec_platform_ruby_abis entry, platforms, platform_ruby_abis
+    else
+      spec_platforms entry, platforms
+    end
     spec_authors     entry, spec
     spec_homepage    entry, spec
     spec_license     entry, spec
@@ -237,36 +249,63 @@ module Gem::QueryUtils
     spec_summary     entry, spec
   end
 
-  def entry_versions(entry, name_tuples, platforms, specs)
+  def entry_versions(entry, name_tuples, platforms, platform_ruby_abis, specs)
     return unless options[:versions]
 
     list =
       if platforms.empty? || options[:details]
         name_tuples.map(&:version).uniq
+      elsif ruby_abi_metadata?(platform_ruby_abis)
+        platforms.sort.reverse.flat_map do |version, pls|
+          out = version_label(version, specs)
+          labels = version_platform_labels(version, pls, platform_ruby_abis, label_platform: true)
+          labels.empty? ? [out] : labels.map {|label| "#{out} #{label}" }
+        end
       else
         platforms.sort.reverse.map do |version, pls|
-          out = version.to_s
+          out = version_label(version, specs)
+          labels = version_platform_labels(version, pls, platform_ruby_abis)
+          labels.empty? ? out : "#{out} #{labels.join(" ")}"
+        end
+      end
 
-          if options[:domain] == :local
-            default = specs.any? do |s|
-              !s.is_a?(Gem::Source) && s.version == version && s.default_gem?
-            end
-            out = "default: #{out}" if default
-          end
+    use_multiline_separator = !options[:details] && ruby_abi_metadata?(platform_ruby_abis) && list.length > 1
+    separator = use_multiline_separator ? "\n#{" " * (entry.first.length + 2)}" : ", "
+    entry << " (#{list.join separator})"
+  end
 
-          if pls != [Gem::Platform::RUBY]
-            platform_list = [pls.delete(Gem::Platform::RUBY), *pls.sort].compact
-            out = platform_list.unshift(out).join(" ")
-          end
+  def version_label(version, specs)
+    out = version.to_s
+    return out unless options[:domain] == :local
 
-          out
-        end
-      end
+    default = specs.any? do |s|
+      !s.is_a?(Gem::Source) && s.version == version && s.default_gem?
+    end
+    default ? "default: #{out}" : out
+  end
+
+  def ruby_abi_metadata?(platform_ruby_abis)
+    platform_ruby_abis.values.any? do |ruby_abis_by_platform|
+      ruby_abis_by_platform.values.any?(&:any?)
+    end
+  end
+
+  def version_platform_labels(version, platforms, platform_ruby_abis, label_platform: false)
+    platforms = platforms.uniq
+    return [] if platforms == [Gem::Platform::RUBY]
 
-    entry << " (#{list.join ", "})"
+    platforms = [platforms.delete(Gem::Platform::RUBY), *platforms.sort].compact
+    platforms.map do |platform|
+      ruby_abis = platform_ruby_abis[version][platform].uniq.sort
+      platform_label = label_platform ? "Platform: #{platform}" : platform
+      next platform_label if ruby_abis.empty?
+
+      separator = label_platform ? ", " : " "
+      "#{platform_label}#{separator}Ruby ABI: #{ruby_abis.join(", ")}"
+    end
   end
 
-  def make_entry(entry_tuples, platforms)
+  def make_entry(entry_tuples, platforms, platform_ruby_abis)
     detail_tuple = entry_tuples.first
 
     name_tuples, specs = entry_tuples.flatten.partition do |item|
@@ -275,8 +314,8 @@ module Gem::QueryUtils
 
     entry = [name_tuples.first.name]
 
-    entry_versions(entry, name_tuples, platforms, specs)
-    entry_details(entry, detail_tuple, specs, platforms)
+    entry_versions(entry, name_tuples, platforms, platform_ruby_abis, specs)
+    entry_details(entry, detail_tuple, specs, platforms, platform_ruby_abis)
 
     entry.join
   end
@@ -319,6 +358,7 @@ module Gem::QueryUtils
   end
 
   def spec_platforms(entry, platforms)
+    platforms = platforms.transform_values(&:uniq)
     non_ruby = platforms.any? do |_, pls|
       pls.any? {|pl| pl != Gem::Platform::RUBY }
     end
@@ -326,7 +366,7 @@ module Gem::QueryUtils
     return unless non_ruby
 
     if platforms.length == 1
-      title = platforms.values.length == 1 ? "Platform" : "Platforms"
+      title = platforms.values.first.length == 1 ? "Platform" : "Platforms"
       entry << "    #{title}: #{platforms.values.sort.join(", ")}\n"
     else
       entry << "    Platforms:\n"
@@ -342,6 +382,26 @@ module Gem::QueryUtils
     end
   end
 
+  def spec_platform_ruby_abis(entry, platforms, platform_ruby_abis)
+    entry << "    Platforms:\n"
+
+    platforms.sort.each do |version, pls|
+      labels = version_platform_labels(version, pls, platform_ruby_abis)
+      next if labels.empty?
+
+      if platforms.length == 1
+        labels.each do |label|
+          entry << "        #{label}\n"
+        end
+      else
+        label = "        #{version}: "
+        data = format_text labels.join(", "), 68, label.length
+        data[0, label.length] = label
+        entry << data << "\n"
+      end
+    end
+  end
+
   def spec_summary(entry, spec)
     summary = truncate_text(spec.summary, "the summary for #{spec.full_name}")
     entry << "\n\n" << format_text(summary, 68, 4)