rubygems-update 2.6.0 → 3.5.4

data/bundler/lib/bundler/compact_index_client/cache.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative "gem_parser"
+
+module Bundler
+  class CompactIndexClient
+    class Cache
+      attr_reader :directory
+
+      def initialize(directory)
+        @directory = Pathname.new(directory).expand_path
+        info_roots.each {|dir| mkdir(dir) }
+        mkdir(info_etag_root)
+      end
+
+      def names
+        lines(names_path)
+      end
+
+      def names_path
+        directory.join("names")
+      end
+
+      def names_etag_path
+        directory.join("names.etag")
+      end
+
+      def versions
+        versions_by_name = Hash.new {|hash, key| hash[key] = [] }
+        info_checksums_by_name = {}
+
+        lines(versions_path).each do |line|
+          name, versions_string, info_checksum = line.split(" ", 3)
+          info_checksums_by_name[name] = info_checksum || ""
+          versions_string.split(",") do |version|
+            delete = version.delete_prefix!("-")
+            version = version.split("-", 2).unshift(name)
+            if delete
+              versions_by_name[name].delete(version)
+            else
+              versions_by_name[name] << version
+            end
+          end
+        end
+
+        [versions_by_name, info_checksums_by_name]
+      end
+
+      def versions_path
+        directory.join("versions")
+      end
+
+      def versions_etag_path
+        directory.join("versions.etag")
+      end
+
+      def checksums
+        checksums = {}
+
+        lines(versions_path).each do |line|
+          name, _, checksum = line.split(" ", 3)
+          checksums[name] = checksum
+        end
+
+        checksums
+      end
+
+      def dependencies(name)
+        lines(info_path(name)).map do |line|
+          parse_gem(line)
+        end
+      end
+
+      def info_path(name)
+        name = name.to_s
+        if /[^a-z0-9_-]/.match?(name)
+          name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}"
+          info_roots.last.join(name)
+        else
+          info_roots.first.join(name)
+        end
+      end
+
+      def info_etag_path(name)
+        name = name.to_s
+        info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}")
+      end
+
+      private
+
+      def mkdir(dir)
+        SharedHelpers.filesystem_access(dir) do
+          FileUtils.mkdir_p(dir)
+        end
+      end
+
+      def lines(path)
+        return [] unless path.file?
+        lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n")
+        header = lines.index("---")
+        header ? lines[header + 1..-1] : lines
+      end
+
+      def parse_gem(line)
+        @dependency_parser ||= GemParser.new
+        @dependency_parser.parse(line)
+      end
+
+      def info_roots
+        [
+          directory.join("info"),
+          directory.join("info-special-characters"),
+        ]
+      end
+
+      def info_etag_root
+        directory.join("info-etags")
+      end
+    end
+  end
+end

data/bundler/lib/bundler/compact_index_client/cache_file.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require_relative "../vendored_fileutils"
+require "rubygems/package"
+
+module Bundler
+  class CompactIndexClient
+    # write cache files in a way that is robust to concurrent modifications
+    # if digests are given, the checksums will be verified
+    class CacheFile
+      DEFAULT_FILE_MODE = 0o644
+      private_constant :DEFAULT_FILE_MODE
+
+      class Error < RuntimeError; end
+      class ClosedError < Error; end
+
+      class DigestMismatchError < Error
+        def initialize(digests, expected_digests)
+          super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}."
+        end
+      end
+
+      # Initialize with a copy of the original file, then yield the instance.
+      def self.copy(path, &block)
+        new(path) do |file|
+          file.initialize_digests
+
+          SharedHelpers.filesystem_access(path, :read) do
+            path.open("rb") do |s|
+              file.open {|f| IO.copy_stream(s, f) }
+            end
+          end
+
+          yield file
+        end
+      end
+
+      # Write data to a temp file, then replace the original file with it verifying the digests if given.
+      def self.write(path, data, digests = nil)
+        return unless data
+        new(path) do |file|
+          file.digests = digests
+          file.write(data)
+        end
+      end
+
+      attr_reader :original_path, :path
+
+      def initialize(original_path, &block)
+        @original_path = original_path
+        @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE
+        @path = original_path.sub(/$/, ".#{$$}.tmp")
+        return unless block_given?
+        begin
+          yield self
+        ensure
+          close
+        end
+      end
+
+      def size
+        path.size
+      end
+
+      # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys.
+      def initialize_digests(keys = nil)
+        @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup
+        @digests.transform_values! {|algo_class| SharedHelpers.digest(algo_class).new }
+      end
+
+      # reset the digests so they don't contain any previously read data
+      def reset_digests
+        @digests&.each_value(&:reset)
+      end
+
+      # set the digests that will be verified at the end
+      def digests=(expected_digests)
+        @expected_digests = expected_digests
+
+        if @expected_digests.nil?
+          @digests = nil
+        elsif @digests
+          @digests = @digests.slice(*@expected_digests.keys)
+        else
+          initialize_digests(@expected_digests.keys)
+        end
+      end
+
+      # remove this method when we stop generating md5 digests for legacy etags
+      def md5
+        @digests && @digests["md5"]
+      end
+
+      def digests?
+        @digests&.any?
+      end
+
+      # Open the temp file for writing, reusing original permissions, yielding the IO object.
+      def open(write_mode = "wb", perm = @perm, &block)
+        raise ClosedError, "Cannot reopen closed file" if @closed
+        SharedHelpers.filesystem_access(path, :write) do
+          path.open(write_mode, perm) do |f|
+            yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f
+          end
+        end
+      end
+
+      # Returns false without appending when no digests since appending is too error prone to do without digests.
+      def append(data)
+        return false unless digests?
+        open("a") {|f| f.write data }
+        verify && commit
+      end
+
+      def write(data)
+        reset_digests
+        open {|f| f.write data }
+        commit!
+      end
+
+      def commit!
+        verify || raise(DigestMismatchError.new(@base64digests, @expected_digests))
+        commit
+      end
+
+      # Verify the digests, returning true on match, false on mismatch.
+      def verify
+        return true unless @expected_digests && digests?
+        @base64digests = @digests.transform_values!(&:base64digest)
+        @digests = nil
+        @base64digests.all? {|algo, digest| @expected_digests[algo] == digest }
+      end
+
+      # Replace the original file with the temp file without verifying digests.
+      # The file is permanently closed.
+      def commit
+        raise ClosedError, "Cannot commit closed file" if @closed
+        SharedHelpers.filesystem_access(original_path, :write) do
+          FileUtils.mv(path, original_path)
+        end
+        @closed = true
+      end
+
+      # Remove the temp file without replacing the original file.
+      # The file is permanently closed.
+      def close
+        return if @closed
+        FileUtils.remove_file(path) if @path&.file?
+        @closed = true
+      end
+    end
+  end
+end

data/bundler/lib/bundler/compact_index_client/gem_parser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    if defined?(Gem::Resolver::APISet::GemParser)
+      GemParser = Gem::Resolver::APISet::GemParser
+    else
+      class GemParser
+        EMPTY_ARRAY = [].freeze
+        private_constant :EMPTY_ARRAY
+
+        def parse(line)
+          version_and_platform, rest = line.split(" ", 2)
+          version, platform = version_and_platform.split("-", 2)
+          dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest
+          dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
+          requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
+          [version, platform, dependencies, requirements]
+        end
+
+        private
+
+        def parse_dependency(string)
+          dependency = string.split(":")
+          dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+          dependency[0] = -dependency[0]
+          dependency
+        end
+      end
+    end
+  end
+end

data/bundler/lib/bundler/compact_index_client/updater.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    class Updater
+      class MismatchedChecksumError < Error
+        def initialize(path, message)
+          super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}"
+        end
+      end
+
+      def initialize(fetcher)
+        @fetcher = fetcher
+      end
+
+      def update(remote_path, local_path, etag_path)
+        append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path)
+      rescue CacheFile::DigestMismatchError => e
+        raise MismatchedChecksumError.new(remote_path, e.message)
+      rescue Zlib::GzipFile::Error
+        raise Bundler::HTTPError
+      end
+
+      private
+
+      def append(remote_path, local_path, etag_path)
+        return false unless local_path.file? && local_path.size.nonzero?
+
+        CacheFile.copy(local_path) do |file|
+          etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+          etag ||= generate_etag(etag_path, file) # Remove this after 2.5.0 has been out for a while.
+
+          # Subtract a byte to ensure the range won't be empty.
+          # Avoids 416 (Range Not Satisfiable) responses.
+          response = @fetcher.call(remote_path, request_headers(etag, file.size - 1))
+          break true if response.is_a?(Gem::Net::HTTPNotModified)
+
+          file.digests = parse_digests(response)
+          # server may ignore Range and return the full response
+          if response.is_a?(Gem::Net::HTTPPartialContent)
+            break false unless file.append(response.body.byteslice(1..-1))
+          else
+            file.write(response.body)
+          end
+          CacheFile.write(etag_path, etag(response))
+          true
+        end
+      end
+
+      # request without range header to get the full file or a 304 Not Modified
+      def replace(remote_path, local_path, etag_path)
+        etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+        response = @fetcher.call(remote_path, request_headers(etag))
+        return true if response.is_a?(Gem::Net::HTTPNotModified)
+        CacheFile.write(local_path, response.body, parse_digests(response))
+        CacheFile.write(etag_path, etag(response))
+      end
+
+      def request_headers(etag, range_start = nil)
+        headers = {}
+        headers["Range"] = "bytes=#{range_start}-" if range_start
+        headers["If-None-Match"] = etag if etag
+        headers
+      end
+
+      def etag_for_request(etag_path)
+        etag_path.read.tap(&:chomp!) if etag_path.file?
+      end
+
+      # When first releasing this opaque etag feature, we want to generate the old MD5 etag
+      # based on the content of the file. After that it will always use the saved opaque etag.
+      # This transparently saves existing users with good caches from updating a bunch of files.
+      # Remove this behavior after 2.5.0 has been out for a while.
+      def generate_etag(etag_path, file)
+        etag = file.md5.hexdigest
+        CacheFile.write(etag_path, etag)
+        etag
+      end
+
+      def etag(response)
+        return unless response["ETag"]
+        etag = response["ETag"].delete_prefix("W/")
+        return if etag.delete_prefix!('"') && !etag.delete_suffix!('"')
+        etag
+      end
+
+      # Unwraps and returns a Hash of digest algorithms and base64 values
+      # according to RFC 8941 Structured Field Values for HTTP.
+      # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence
+      # Ignores unsupported algorithms.
+      def parse_digests(response)
+        return unless header = response["Repr-Digest"] || response["Digest"]
+        digests = {}
+        header.split(",") do |param|
+          algorithm, value = param.split("=", 2)
+          algorithm.strip!
+          algorithm.downcase!
+          next unless SUPPORTED_DIGESTS.key?(algorithm)
+          next unless value = byte_sequence(value)
+          digests[algorithm] = value
+        end
+        digests.empty? ? nil : digests
+      end
+
+      # Unwrap surrounding colons (byte sequence)
+      # The wrapping characters must be matched or we return nil.
+      # Also handles quotes because right now rubygems.org sends them.
+      def byte_sequence(value)
+        return if value.delete_prefix!(":") && !value.delete_suffix!(":")
+        return if value.delete_prefix!('"') && !value.delete_suffix!('"')
+        value
+      end
+    end
+  end
+end

data/bundler/lib/bundler/compact_index_client.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "set"
+
+module Bundler
+  class CompactIndexClient
+    # NOTE: MD5 is here not because we expect a server to respond with it, but
+    # because we use it to generate the etag on first request during the upgrade
+    # to the compact index client that uses opaque etags saved to files.
+    # Remove once 2.5.0 has been out for a while.
+    SUPPORTED_DIGESTS = { "sha-256" => :SHA256, "md5" => :MD5 }.freeze
+    DEBUG_MUTEX = Thread::Mutex.new
+
+    def self.debug
+      return unless ENV["DEBUG_COMPACT_INDEX"]
+      DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
+    end
+
+    class Error < StandardError; end
+
+    require_relative "compact_index_client/cache"
+    require_relative "compact_index_client/cache_file"
+    require_relative "compact_index_client/updater"
+
+    attr_reader :directory
+
+    def initialize(directory, fetcher)
+      @directory = Pathname.new(directory)
+      @updater = Updater.new(fetcher)
+      @cache = Cache.new(@directory)
+      @endpoints = Set.new
+      @info_checksums_by_name = {}
+      @parsed_checksums = false
+      @mutex = Thread::Mutex.new
+    end
+
+    def execution_mode=(block)
+      Bundler::CompactIndexClient.debug { "execution_mode=" }
+      @endpoints = Set.new
+
+      @execution_mode = block
+    end
+
+    # @return [Lambda] A lambda that takes an array of inputs and a block, and
+    #         maps the inputs with the block in parallel.
+    #
+    def execution_mode
+      @execution_mode || sequentially
+    end
+
+    def sequential_execution_mode!
+      self.execution_mode = sequentially
+    end
+
+    def sequentially
+      @sequentially ||= lambda do |inputs, &blk|
+        inputs.map(&blk)
+      end
+    end
+
+    def names
+      Bundler::CompactIndexClient.debug { "/names" }
+      update("names", @cache.names_path, @cache.names_etag_path)
+      @cache.names
+    end
+
+    def versions
+      Bundler::CompactIndexClient.debug { "/versions" }
+      update("versions", @cache.versions_path, @cache.versions_etag_path)
+      versions, @info_checksums_by_name = @cache.versions
+      versions
+    end
+
+    def dependencies(names)
+      Bundler::CompactIndexClient.debug { "dependencies(#{names})" }
+      execution_mode.call(names) do |name|
+        update_info(name)
+        @cache.dependencies(name).map {|d| d.unshift(name) }
+      end.flatten(1)
+    end
+
+    def update_and_parse_checksums!
+      Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" }
+      return @info_checksums_by_name if @parsed_checksums
+      update("versions", @cache.versions_path, @cache.versions_etag_path)
+      @info_checksums_by_name = @cache.checksums
+      @parsed_checksums = true
+    end
+
+    private
+
+    def update(remote_path, local_path, local_etag_path)
+      Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" }
+      unless synchronize { @endpoints.add?(remote_path) }
+        Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" }
+        return
+      end
+      @updater.update(url(remote_path), local_path, local_etag_path)
+    end
+
+    def update_info(name)
+      Bundler::CompactIndexClient.debug { "update_info(#{name})" }
+      path = @cache.info_path(name)
+      unless existing = @info_checksums_by_name[name]
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" }
+        return
+      end
+      checksum = SharedHelpers.checksum_for_file(path, :MD5)
+      if checksum == existing
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" }
+        return
+      end
+      Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" }
+      update("info/#{name}", path, @cache.info_etag_path(name))
+    end
+
+    def url(path)
+      path
+    end
+
+    def synchronize
+      @mutex.synchronize { yield }
+    end
+  end
+end

data/bundler/lib/bundler/constants.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bundler
+  WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/
+  FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd")
+  NULL    = File::NULL
+end

data/bundler/lib/bundler/current_ruby.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Bundler
+  # Returns current version of Ruby
+  #
+  # @return [CurrentRuby] Current version of Ruby
+  def self.current_ruby
+    @current_ruby ||= CurrentRuby.new
+  end
+
+  class CurrentRuby
+    KNOWN_MINOR_VERSIONS = %w[
+      1.8
+      1.9
+      2.0
+      2.1
+      2.2
+      2.3
+      2.4
+      2.5
+      2.6
+      2.7
+      3.0
+      3.1
+      3.2
+      3.3
+    ].freeze
+
+    KNOWN_MAJOR_VERSIONS = KNOWN_MINOR_VERSIONS.map {|v| v.split(".", 2).first }.uniq.freeze
+
+    KNOWN_PLATFORMS = %w[
+      jruby
+      maglev
+      mingw
+      mri
+      mswin
+      mswin64
+      rbx
+      ruby
+      truffleruby
+      windows
+      x64_mingw
+    ].freeze
+
+    def ruby?
+      return true if Bundler::GemHelpers.generic_local_platform_is_ruby?
+
+      !windows? && (RUBY_ENGINE == "ruby" || RUBY_ENGINE == "rbx" || RUBY_ENGINE == "maglev" || RUBY_ENGINE == "truffleruby")
+    end
+
+    def mri?
+      !windows? && RUBY_ENGINE == "ruby"
+    end
+
+    def rbx?
+      ruby? && RUBY_ENGINE == "rbx"
+    end
+
+    def jruby?
+      RUBY_ENGINE == "jruby"
+    end
+
+    def maglev?
+      RUBY_ENGINE == "maglev"
+    end
+
+    def truffleruby?
+      RUBY_ENGINE == "truffleruby"
+    end
+
+    def windows?
+      Gem.win_platform?
+    end
+    alias_method :mswin?, :windows?
+    alias_method :mswin64?, :windows?
+    alias_method :mingw?, :windows?
+    alias_method :x64_mingw?, :windows?
+
+    (KNOWN_MINOR_VERSIONS + KNOWN_MAJOR_VERSIONS).each do |version|
+      trimmed_version = version.tr(".", "")
+      define_method(:"on_#{trimmed_version}?") do
+        RUBY_VERSION.start_with?("#{version}.")
+      end
+
+      KNOWN_PLATFORMS.each do |platform|
+        define_method(:"#{platform}_#{trimmed_version}?") do
+          send(:"#{platform}?") && send(:"on_#{trimmed_version}?")
+        end
+      end
+    end
+  end
+end