rubygems-update 4.0.21 → 4.1.0.beta1

data/lib/rubygems/compact_index_client/cache.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "pathname" unless defined?(Pathname)
+require "set"
+
+class Gem::CompactIndexClient
+  # Calls the Updater to update the cached files on disk, reads the
+  # cached files and returns their contents.
+  class Cache
+    attr_reader :directory
+
+    def initialize(directory, fetcher = nil)
+      @directory = Pathname.new(directory).expand_path
+      @updater = Updater.new(fetcher) if fetcher
+      @mutex = Thread::Mutex.new
+      @endpoints = Set.new
+
+      @info_root = mkdir("info")
+      @special_characters_info_root = mkdir("info-special-characters")
+      @info_etag_root = mkdir("info-etags")
+    end
+
+    def names
+      fetch("names", names_path, names_etag_path)
+    end
+
+    def versions
+      fetch("versions", versions_path, versions_etag_path)
+    end
+
+    def info(name, remote_checksum = nil)
+      path = info_path(name)
+
+      if remote_checksum && remote_checksum != checksum_for_file(path)
+        fetch("info/#{name}", path, info_etag_path(name))
+      else
+        Gem::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum matches local" : "versions index checksum is nil"})" }
+        read(path)
+      end
+    end
+
+    # Fetch a single gem's info file without consulting the versions
+    # index, refreshing the cached file with a conditional request.
+    def fetch_info(name)
+      fetch("info/#{name}", info_path(name), info_etag_path(name))
+    end
+
+    def reset!
+      @mutex.synchronize { @endpoints.clear }
+    end
+
+    private
+
+    def names_path = directory.join("names")
+    def names_etag_path = directory.join("names.etag")
+    def versions_path = directory.join("versions")
+    def versions_etag_path = directory.join("versions.etag")
+
+    def info_path(name)
+      name = name.to_s
+      validate_name!(name)
+      if /[^a-z0-9_-]/.match?(name)
+        name += "-#{Digest::MD5.hexdigest(name).downcase}"
+        @special_characters_info_root.join(name)
+      else
+        @info_root.join(name)
+      end
+    end
+
+    def info_etag_path(name)
+      name = name.to_s
+      validate_name!(name)
+      @info_etag_root.join("#{name}-#{Digest::MD5.hexdigest(name).downcase}")
+    end
+
+    # Gem names come from the remote index and are not otherwise
+    # validated, so refuse anything that would escape the cache
+    # directory when used as a path component.
+    def validate_name!(name)
+      return unless name.empty? || name.include?("\0") || name == "." || name == ".." || File.basename(name) != name
+
+      raise Gem::Exception, "malformed gem name: #{name.inspect}"
+    end
+
+    def checksum_for_file(path)
+      return unless path.file?
+      Gem::CompactIndexClient.filesystem_access(path, :read) do
+        Digest::MD5.file(path).hexdigest
+      end
+    end
+
+    def mkdir(name)
+      directory.join(name).tap do |dir|
+        Gem::CompactIndexClient.filesystem_access(dir) do
+          FileUtils.mkdir_p(dir)
+        end
+      end
+    end
+
+    def fetch(remote_path, path, etag_path)
+      if already_fetched?(remote_path)
+        Gem::CompactIndexClient.debug { "already fetched #{remote_path}" }
+      else
+        Gem::CompactIndexClient.debug { "fetching #{remote_path}" }
+        @updater&.update(remote_path, path, etag_path)
+      end
+
+      read(path)
+    end
+
+    def already_fetched?(remote_path)
+      @mutex.synchronize { !@endpoints.add?(remote_path) }
+    end
+
+    def read(path)
+      return unless path.file?
+      Gem::CompactIndexClient.filesystem_access(path, :read, &:read)
+    end
+  end
+end

data/lib/rubygems/compact_index_client/cache_file.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "rubygems/package"
+
+class Gem::CompactIndexClient
+  # write cache files in a way that is robust to concurrent modifications
+  # if digests are given, the checksums will be verified
+  class CacheFile
+    DEFAULT_FILE_MODE = 0o644
+    private_constant :DEFAULT_FILE_MODE
+
+    class Error < RuntimeError; end
+    class ClosedError < Error; end
+
+    class DigestMismatchError < Error
+      def initialize(digests, expected_digests)
+        super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}."
+      end
+    end
+
+    # Initialize with a copy of the original file, then yield the instance.
+    def self.copy(path, &block)
+      new(path) do |file|
+        file.initialize_digests
+
+        Gem::CompactIndexClient.filesystem_access(path, :read) do
+          path.open("rb") do |s|
+            file.open {|f| IO.copy_stream(s, f) }
+          end
+        end
+
+        yield file
+      end
+    end
+
+    # Write data to a temp file, then replace the original file with it verifying the digests if given.
+    def self.write(path, data, digests = nil)
+      return unless data
+      new(path) do |file|
+        file.digests = digests
+        file.write(data)
+      end
+    end
+
+    attr_reader :original_path, :path
+
+    def initialize(original_path, &block)
+      @original_path = original_path
+      @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE
+      @path = original_path.sub(/$/, ".#{$$}.tmp")
+      return unless block_given?
+      begin
+        yield self
+      ensure
+        close
+      end
+    end
+
+    def size
+      path.size
+    end
+
+    # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys.
+    def initialize_digests(keys = nil)
+      @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup
+      @digests.transform_values! {|algo_class| Digest(algo_class).new }
+    end
+
+    # reset the digests so they don't contain any previously read data
+    def reset_digests
+      @digests&.each_value(&:reset)
+    end
+
+    # set the digests that will be verified at the end
+    def digests=(expected_digests)
+      @expected_digests = expected_digests
+
+      if @expected_digests.nil?
+        @digests = nil
+      elsif @digests
+        @digests = @digests.slice(*@expected_digests.keys)
+      else
+        initialize_digests(@expected_digests.keys)
+      end
+    end
+
+    def digests?
+      @digests&.any?
+    end
+
+    # Open the temp file for writing, reusing original permissions, yielding the IO object.
+    def open(write_mode = "wb", perm = @perm, &block)
+      raise ClosedError, "Cannot reopen closed file" if @closed
+      Gem::CompactIndexClient.filesystem_access(path, :write) do
+        path.open(write_mode, perm) do |f|
+          yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f
+        end
+      end
+    end
+
+    # Returns false without appending when no digests since appending is too error prone to do without digests.
+    def append(data)
+      return false unless digests?
+      open("ab") {|f| f.write data }
+      verify && commit
+    end
+
+    def write(data)
+      reset_digests
+      open {|f| f.write data }
+      commit!
+    end
+
+    def commit!
+      verify || raise(DigestMismatchError.new(@base64digests, @expected_digests))
+      commit
+    end
+
+    # Verify the digests, returning true on match, false on mismatch.
+    def verify
+      return true unless @expected_digests && digests?
+      @base64digests = @digests.transform_values!(&:base64digest)
+      @digests = nil
+      @base64digests.all? {|algo, digest| @expected_digests[algo] == digest }
+    end
+
+    # Replace the original file with the temp file without verifying digests.
+    # The file is permanently closed.
+    def commit
+      raise ClosedError, "Cannot commit closed file" if @closed
+      Gem::CompactIndexClient.filesystem_access(original_path, :write) do
+        FileUtils.mv(path, original_path)
+      end
+      @closed = true
+    end
+
+    # Remove the temp file without replacing the original file.
+    # The file is permanently closed.
+    def close
+      return if @closed
+      FileUtils.remove_file(path) if @path&.file?
+      @closed = true
+    end
+  end
+end

data/lib/rubygems/compact_index_client/http_fetcher.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "rubygems/remote_fetcher"
+
+class Gem::CompactIndexClient
+  # Fetches compact index files relative to +base_uri+ using
+  # Gem::RemoteFetcher's connection infrastructure (proxy, TLS,
+  # connection pooling). Implements the fetcher interface expected by
+  # Gem::CompactIndexClient: #call(path, headers) returning a
+  # Gem::Net::HTTP response.
+  class HTTPFetcher
+    REDIRECT_LIMIT = 10
+    private_constant :REDIRECT_LIMIT
+
+    def initialize(base_uri, remote_fetcher = Gem::RemoteFetcher.fetcher)
+      base_uri = base_uri.to_s
+      base_uri += "/" unless base_uri.end_with?("/")
+      @base_uri = Gem::URI(base_uri)
+      @remote_fetcher = remote_fetcher
+    end
+
+    def call(path, headers = {})
+      fetch(@base_uri + path, headers, REDIRECT_LIMIT)
+    end
+
+    private
+
+    def fetch(uri, headers, redirects_remaining)
+      response = request(uri, headers)
+
+      case response
+      when Gem::Net::HTTPNotModified
+        response
+      when Gem::Net::HTTPSuccess
+        # The callers write the body into the cache, so a body-less success
+        # such as 204 would truncate the cached file.
+        raise bad_response(response, uri) unless response.class.body_permitted?
+
+        response
+      when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther,
+           Gem::Net::HTTPTemporaryRedirect, Gem::Net::HTTPPermanentRedirect
+        raise Gem::RemoteFetcher::FetchError.new("too many redirects", uri) if redirects_remaining.zero?
+
+        location = response["Location"]
+        raise Gem::RemoteFetcher::FetchError.new("redirecting but no redirect location was given", uri) unless location
+
+        redirect = uri + location
+        if https?(uri) && !https?(redirect)
+          raise Gem::RemoteFetcher::FetchError.new("redirecting to non-https resource: #{Gem::Uri.redact(redirect)}", uri)
+        end
+        # An absolute Location on the same host drops the credentials that a
+        # relative one would have kept.
+        redirect.userinfo = uri.userinfo if redirect.host == uri.host && !redirect.userinfo
+
+        fetch(redirect, headers, redirects_remaining - 1)
+      when Gem::Net::HTTPRangeNotSatisfiable
+        raise bad_response(response, uri) unless headers.key?("Range")
+
+        # The local cache is longer than the remote file, refetch it whole. A
+        # matching ETag would otherwise turn the retry into a 304 and keep the
+        # oversized cache.
+        fetch(uri, headers.except("Range", "If-None-Match"), redirects_remaining)
+      else
+        raise bad_response(response, uri)
+      end
+    end
+
+    # The callers fall back to the Marshal index when a fetch fails, and they
+    # only recognize a failure that arrives as a FetchError.
+    def request(uri, headers)
+      @remote_fetcher.request(uri, Gem::Net::HTTP::Get) do |req|
+        headers.each {|name, value| req[name] = value }
+      end
+    rescue Gem::Timeout::Error, IOError, SocketError, SystemCallError,
+           *(OpenSSL::SSL::SSLError if Gem::HAVE_OPENSSL) => e
+      raise Gem::RemoteFetcher::FetchError.new("#{e.class}: #{e}", uri)
+    end
+
+    def bad_response(response, uri)
+      detail = response["X-Error-Message"] || response.message
+      Gem::RemoteFetcher::FetchError.new("bad response #{detail} #{response.code}", uri)
+    end
+
+    def https?(uri)
+      uri.scheme == "https"
+    end
+  end
+end

data/lib/rubygems/compact_index_client/parser.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "rubygems/resolver/api_set/gem_parser"
+
+class Gem::CompactIndexClient
+  class Parser
+    # `compact_index` - an object responding to #names, #versions, #info(name, checksum),
+    #                   returning the file contents as a string
+    def initialize(compact_index)
+      @compact_index = compact_index
+      @info_checksums = nil
+      @versions_by_name = nil
+      @available = nil
+      @gem_parser = nil
+    end
+
+    def names
+      lines(@compact_index.names)
+    end
+
+    def versions
+      @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] }
+      @info_checksums = {}
+
+      lines(@compact_index.versions).each do |line|
+        name, versions_string, checksum = line.split(" ", 3)
+        @info_checksums[name] = checksum || ""
+        versions_string.split(",") do |version|
+          delete = version.delete_prefix!("-")
+          version = version.split("-", 2).unshift(name)
+          if delete
+            @versions_by_name[name].delete(version)
+          else
+            @versions_by_name[name] << version
+          end
+        end
+      end
+
+      @versions_by_name
+    end
+
+    def info(name)
+      parse_info(@compact_index.info(name, info_checksums[name]), name)
+    end
+
+    def parse_info(data, name)
+      lines(data).map {|line| gem_parser.parse(line).unshift(name) }
+    end
+
+    def available?
+      return @available unless @available.nil?
+      @available = !info_checksums.empty?
+    end
+
+    private
+
+    def info_checksums
+      @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums|
+        parse_version_checksum(line, checksums)
+      end
+    end
+
+    def lines(data)
+      return [] if data.nil? || data.empty?
+      lines = data.split("\n")
+      header = lines.index("---")
+      header ? lines[header + 1..-1] : lines
+    end
+
+    def gem_parser
+      @gem_parser ||= Gem::Resolver::APISet::GemParser.new
+    end
+
+    # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects.
+    # This method gets called at least once for every gem when parsing versions.
+    def parse_version_checksum(line, checksums)
+      return unless (name_end = line.index(" ")) # Artifactory bug causes blank lines in artifactory index files
+      checksum_start = line.index(" ", name_end + 1)
+      return unless checksum_start
+      checksum_start += 1
+
+      checksum_end = line.size - checksum_start
+
+      line.freeze # allows slicing into the string to not allocate a copy of the line
+      name = line[0, name_end]
+      checksum = line[checksum_start, checksum_end]
+      checksums[name.freeze] = checksum # freeze name since it is used as a hash key
+    end
+  end
+end

data/lib/rubygems/compact_index_client/updater.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+# On RubyGems versions too old to vendor Gem::Net (when this file is loaded
+# from the copy shipped inside the Bundler gem), Bundler's own
+# vendored_net_http shim has already defined it before this file is loaded.
+begin
+  require "rubygems/vendored_net_http" unless defined?(Gem::Net::HTTP)
+rescue LoadError
+  require "net/http"
+end
+
+class Gem::CompactIndexClient
+  # Updates the cached files on disk, keeping them in sync with the server
+  # using conditional requests (ETag) and ranged requests where possible.
+  class Updater
+    class MismatchedChecksumError < Error
+      def initialize(path, message)
+        super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}"
+      end
+    end
+
+    def initialize(fetcher)
+      @fetcher = fetcher
+    end
+
+    def update(remote_path, local_path, etag_path)
+      append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path)
+    rescue CacheFile::DigestMismatchError => e
+      raise MismatchedChecksumError.new(remote_path, e.message)
+    rescue Zlib::GzipFile::Error
+      raise Error, "invalid gzip response while fetching /#{remote_path}"
+    end
+
+    private
+
+    def append(remote_path, local_path, etag_path)
+      return false unless local_path.file? && local_path.size.nonzero?
+
+      CacheFile.copy(local_path) do |file|
+        etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+
+        # Subtract a byte to ensure the range won't be empty.
+        # Avoids 416 (Range Not Satisfiable) responses.
+        response = @fetcher.call(remote_path, request_headers(etag, file.size - 1))
+        break true if response.is_a?(Gem::Net::HTTPNotModified)
+
+        file.digests = parse_digests(response)
+        # server may ignore Range and return the full response
+        if response.is_a?(Gem::Net::HTTPPartialContent)
+          tail = response.body.byteslice(1..-1)
+          break false unless tail && file.append(tail)
+        else
+          file.write(response.body)
+        end
+        CacheFile.write(etag_path, etag_from_response(response))
+        true
+      end
+    end
+
+    # request without range header to get the full file or a 304 Not Modified
+    def replace(remote_path, local_path, etag_path)
+      etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+      response = @fetcher.call(remote_path, request_headers(etag))
+      return true if response.is_a?(Gem::Net::HTTPNotModified)
+      CacheFile.write(local_path, response.body, parse_digests(response))
+      CacheFile.write(etag_path, etag_from_response(response))
+    end
+
+    def request_headers(etag, range_start = nil)
+      headers = {}
+      headers["Range"] = "bytes=#{range_start}-" if range_start
+      headers["If-None-Match"] = %("#{etag}") if etag
+      headers
+    end
+
+    def etag_from_response(response)
+      return unless response["ETag"]
+      etag = response["ETag"].delete_prefix("W/")
+      return if etag.delete_prefix!('"') && !etag.delete_suffix!('"')
+      etag
+    end
+
+    # Unwraps and returns a Hash of digest algorithms and base64 values
+    # according to RFC 8941 Structured Field Values for HTTP.
+    # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence
+    # Ignores unsupported algorithms.
+    def parse_digests(response)
+      return unless header = response["Repr-Digest"] || response["Digest"]
+      digests = {}
+      header.split(",") do |param|
+        algorithm, value = param.split("=", 2)
+        algorithm.strip!
+        algorithm.downcase!
+        next unless SUPPORTED_DIGESTS.key?(algorithm)
+        next unless value
+        next unless value = byte_sequence(value)
+        digests[algorithm] = value
+      end
+      digests.empty? ? nil : digests
+    end
+
+    # Unwrap surrounding colons (byte sequence)
+    # The wrapping characters must be matched or we return nil.
+    # Also handles quotes because right now rubygems.org sends them.
+    def byte_sequence(value)
+      return if value.delete_prefix!(":") && !value.delete_suffix!(":")
+      return if value.delete_prefix!('"') && !value.delete_suffix!('"')
+      value
+    end
+  end
+end

data/lib/rubygems/compact_index_client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# Skip reloading when an identical copy (e.g. the one shipped inside the Bundler
+# gem) was already required from a different path, to avoid redefinition warnings.
+return if defined?(Gem::CompactIndexClient::INFO_REQS)
+
+##
+# The CompactIndexClient fetches and parses the compact index files
+# (names, versions and info/[gem]) served by a gem server, keeping a
+# local cache so subsequent fetches only transfer what changed.
+#
+# This is the single shared client: Bundler ships a copy of it and uses
+# it underneath its own fetcher orchestration, injecting its fetcher and
+# filesystem access hook.
+
+class Gem::CompactIndexClient
+  SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze
+  DEBUG_MUTEX = Thread::Mutex.new
+
+  # info returns an Array of INFO Arrays. Each INFO Array has the following indices:
+  INFO_NAME = 0
+  INFO_VERSION = 1
+  INFO_SUFFIX = 2
+  INFO_DEPS = 3
+  INFO_REQS = 4
+
+  INFO_PLATFORM = INFO_SUFFIX
+  deprecate_constant :INFO_PLATFORM
+
+  def self.debug
+    return unless ENV["DEBUG_COMPACT_INDEX"]
+    DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
+  end
+
+  class Error < StandardError; end
+
+  # Filesystem reads and writes funnel through this hook so that a host
+  # (e.g. Bundler) can translate low-level Errno exceptions into its own
+  # friendlier errors. The default just yields the path.
+  def self.filesystem_access(path, action = :write, &block)
+    if @filesystem_access
+      @filesystem_access.call(path, action, &block)
+    else
+      yield path
+    end
+  end
+
+  def self.filesystem_access=(hook)
+    @filesystem_access = hook
+  end
+
+  require_relative "compact_index_client/cache"
+  require_relative "compact_index_client/cache_file"
+  require_relative "compact_index_client/http_fetcher"
+  require_relative "compact_index_client/parser"
+  require_relative "compact_index_client/updater"
+
+  # The client is instantiated with:
+  # - `directory`: the root directory where the cache files are stored.
+  # - `fetcher`: (optional) an object that responds to #call(uri_path, headers)
+  #   and returns a Gem::Net::HTTP response. When the fetcher is not provided,
+  #   the client only reads cached files from disk.
+  def initialize(directory, fetcher = nil)
+    @cache = Cache.new(directory, fetcher)
+    @parser = Parser.new(@cache)
+  end
+
+  def names
+    Gem::CompactIndexClient.debug { "names" }
+    @parser.names
+  end
+
+  def versions
+    Gem::CompactIndexClient.debug { "versions" }
+    @parser.versions
+  end
+
+  def dependencies(names)
+    Gem::CompactIndexClient.debug { "dependencies(#{names})" }
+    names.map {|name| info(name) }
+  end
+
+  def info(name)
+    Gem::CompactIndexClient.debug { "info(#{name})" }
+    @parser.info(name)
+  end
+
+  # Fetches a single gem's info without consulting the versions index,
+  # using a conditional request to refresh the cached file. Useful when
+  # only a few gems are needed and the versions index download would
+  # dominate, as in gem install.
+  def fetch_info(name)
+    Gem::CompactIndexClient.debug { "fetch_info(#{name})" }
+    @parser.parse_info(@cache.fetch_info(name), name)
+  end
+
+  def latest_version(name)
+    Gem::CompactIndexClient.debug { "latest_version(#{name})" }
+    @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max
+  end
+
+  def available?
+    Gem::CompactIndexClient.debug { "available?" }
+    @parser.available?
+  end
+
+  def reset!
+    Gem::CompactIndexClient.debug { "reset!" }
+    @cache.reset!
+  end
+end