bundler-plus 4.0.15

data/lib/bundler/compact_index_client/cache.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "rubygems/resolver/api_set/gem_parser"
+
+module Bundler
+  class CompactIndexClient
+    class Cache
+      attr_reader :directory
+
+      def initialize(directory, fetcher = nil)
+        @directory = Pathname.new(directory).expand_path
+        @updater = Updater.new(fetcher) if fetcher
+        @mutex = Thread::Mutex.new
+        @endpoints = Set.new
+
+        @info_root = mkdir("info")
+        @special_characters_info_root = mkdir("info-special-characters")
+        @info_etag_root = mkdir("info-etags")
+      end
+
+      def names
+        fetch("names", names_path, names_etag_path)
+      end
+
+      def versions
+        fetch("versions", versions_path, versions_etag_path)
+      end
+
+      def info(name, remote_checksum = nil)
+        path = info_path(name)
+
+        if remote_checksum && remote_checksum != SharedHelpers.checksum_for_file(path, :MD5)
+          fetch("info/#{name}", path, info_etag_path(name))
+        else
+          Bundler::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum is nil" : "versions index checksum matches local"})" }
+          read(path)
+        end
+      end
+
+      def reset!
+        @mutex.synchronize { @endpoints.clear }
+      end
+
+      private
+
+      def names_path = directory.join("names")
+      def names_etag_path = directory.join("names.etag")
+      def versions_path = directory.join("versions")
+      def versions_etag_path = directory.join("versions.etag")
+
+      def info_path(name)
+        name = name.to_s
+        # TODO: converge this into the info_root by hashing all filenames like info_etag_path
+        if /[^a-z0-9_-]/.match?(name)
+          name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}"
+          @special_characters_info_root.join(name)
+        else
+          @info_root.join(name)
+        end
+      end
+
+      def info_etag_path(name)
+        name = name.to_s
+        @info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}")
+      end
+
+      def mkdir(name)
+        directory.join(name).tap do |dir|
+          SharedHelpers.filesystem_access(dir) do
+            FileUtils.mkdir_p(dir)
+          end
+        end
+      end
+
+      def fetch(remote_path, path, etag_path)
+        if already_fetched?(remote_path)
+          Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" }
+        else
+          Bundler::CompactIndexClient.debug { "fetching #{remote_path}" }
+          @updater&.update(remote_path, path, etag_path)
+        end
+
+        read(path)
+      end
+
+      def already_fetched?(remote_path)
+        @mutex.synchronize { !@endpoints.add?(remote_path) }
+      end
+
+      def read(path)
+        return unless path.file?
+        SharedHelpers.filesystem_access(path, :read, &:read)
+      end
+    end
+  end
+end

data/lib/bundler/compact_index_client/cache_file.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative "../vendored_fileutils"
+require "rubygems/package"
+
+module Bundler
+  class CompactIndexClient
+    # write cache files in a way that is robust to concurrent modifications
+    # if digests are given, the checksums will be verified
+    class CacheFile
+      DEFAULT_FILE_MODE = 0o644
+      private_constant :DEFAULT_FILE_MODE
+
+      class Error < RuntimeError; end
+      class ClosedError < Error; end
+
+      class DigestMismatchError < Error
+        def initialize(digests, expected_digests)
+          super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}."
+        end
+      end
+
+      # Initialize with a copy of the original file, then yield the instance.
+      def self.copy(path, &block)
+        new(path) do |file|
+          file.initialize_digests
+
+          SharedHelpers.filesystem_access(path, :read) do
+            path.open("rb") do |s|
+              file.open {|f| IO.copy_stream(s, f) }
+            end
+          end
+
+          yield file
+        end
+      end
+
+      # Write data to a temp file, then replace the original file with it verifying the digests if given.
+      def self.write(path, data, digests = nil)
+        return unless data
+        new(path) do |file|
+          file.digests = digests
+          file.write(data)
+        end
+      end
+
+      attr_reader :original_path, :path
+
+      def initialize(original_path, &block)
+        @original_path = original_path
+        @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE
+        @path = original_path.sub(/$/, ".#{$$}.tmp")
+        return unless block_given?
+        begin
+          yield self
+        ensure
+          close
+        end
+      end
+
+      def size
+        path.size
+      end
+
+      # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys.
+      def initialize_digests(keys = nil)
+        @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup
+        @digests.transform_values! {|algo_class| SharedHelpers.digest(algo_class).new }
+      end
+
+      # reset the digests so they don't contain any previously read data
+      def reset_digests
+        @digests&.each_value(&:reset)
+      end
+
+      # set the digests that will be verified at the end
+      def digests=(expected_digests)
+        @expected_digests = expected_digests
+
+        if @expected_digests.nil?
+          @digests = nil
+        elsif @digests
+          @digests = @digests.slice(*@expected_digests.keys)
+        else
+          initialize_digests(@expected_digests.keys)
+        end
+      end
+
+      def digests?
+        @digests&.any?
+      end
+
+      # Open the temp file for writing, reusing original permissions, yielding the IO object.
+      def open(write_mode = "wb", perm = @perm, &block)
+        raise ClosedError, "Cannot reopen closed file" if @closed
+        SharedHelpers.filesystem_access(path, :write) do
+          path.open(write_mode, perm) do |f|
+            yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f
+          end
+        end
+      end
+
+      # Returns false without appending when no digests since appending is too error prone to do without digests.
+      def append(data)
+        return false unless digests?
+        open("a") {|f| f.write data }
+        verify && commit
+      end
+
+      def write(data)
+        reset_digests
+        open {|f| f.write data }
+        commit!
+      end
+
+      def commit!
+        verify || raise(DigestMismatchError.new(@base64digests, @expected_digests))
+        commit
+      end
+
+      # Verify the digests, returning true on match, false on mismatch.
+      def verify
+        return true unless @expected_digests && digests?
+        @base64digests = @digests.transform_values!(&:base64digest)
+        @digests = nil
+        @base64digests.all? {|algo, digest| @expected_digests[algo] == digest }
+      end
+
+      # Replace the original file with the temp file without verifying digests.
+      # The file is permanently closed.
+      def commit
+        raise ClosedError, "Cannot commit closed file" if @closed
+        SharedHelpers.filesystem_access(original_path, :write) do
+          FileUtils.mv(path, original_path)
+        end
+        @closed = true
+      end
+
+      # Remove the temp file without replacing the original file.
+      # The file is permanently closed.
+      def close
+        return if @closed
+        FileUtils.remove_file(path) if @path&.file?
+        @closed = true
+      end
+    end
+  end
+end

data/lib/bundler/compact_index_client/parser.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    class Parser
+      # `compact_index` - an object responding to #names, #versions, #info(name, checksum),
+      #                   returning the file contents as a string
+      def initialize(compact_index)
+        @compact_index = compact_index
+        @info_checksums = nil
+        @versions_by_name = nil
+        @available = nil
+        @gem_parser = nil
+      end
+
+      def names
+        lines(@compact_index.names)
+      end
+
+      def versions
+        @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] }
+        @info_checksums = {}
+
+        lines(@compact_index.versions).each do |line|
+          name, versions_string, checksum = line.split(" ", 3)
+          @info_checksums[name] = checksum || ""
+          versions_string.split(",") do |version|
+            delete = version.delete_prefix!("-")
+            version = version.split("-", 2).unshift(name)
+            if delete
+              @versions_by_name[name].delete(version)
+            else
+              @versions_by_name[name] << version
+            end
+          end
+        end
+
+        @versions_by_name
+      end
+
+      def info(name)
+        data = @compact_index.info(name, info_checksums[name])
+        lines(data).map {|line| gem_parser.parse(line).unshift(name) }
+      end
+
+      def available?
+        return @available unless @available.nil?
+        @available = !info_checksums.empty?
+      end
+
+      private
+
+      def info_checksums
+        @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums|
+          parse_version_checksum(line, checksums)
+        end
+      end
+
+      def lines(data)
+        return [] if data.nil? || data.empty?
+        lines = data.split("\n")
+        header = lines.index("---")
+        header ? lines[header + 1..-1] : lines
+      end
+
+      def gem_parser
+        @gem_parser ||= Gem::Resolver::APISet::GemParser.new
+      end
+
+      # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects.
+      # This method gets called at least once for every gem when parsing versions.
+      def parse_version_checksum(line, checksums)
+        return unless (name_end = line.index(" ")) # Artifactory bug causes blank lines in artifactor index files
+        checksum_start = line.index(" ", name_end + 1)
+        return unless checksum_start
+        checksum_start += 1
+
+        checksum_end = line.size - checksum_start
+
+        line.freeze # allows slicing into the string to not allocate a copy of the line
+        name = line[0, name_end]
+        checksum = line[checksum_start, checksum_end]
+        checksums[name.freeze] = checksum # freeze name since it is used as a hash key
+      end
+    end
+  end
+end

data/lib/bundler/compact_index_client/updater.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    class Updater
+      class MismatchedChecksumError < Error
+        def initialize(path, message)
+          super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}"
+        end
+      end
+
+      def initialize(fetcher)
+        @fetcher = fetcher
+      end
+
+      def update(remote_path, local_path, etag_path)
+        append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path)
+      rescue CacheFile::DigestMismatchError => e
+        raise MismatchedChecksumError.new(remote_path, e.message)
+      rescue Zlib::GzipFile::Error
+        raise Bundler::HTTPError
+      end
+
+      private
+
+      def append(remote_path, local_path, etag_path)
+        return false unless local_path.file? && local_path.size.nonzero?
+
+        CacheFile.copy(local_path) do |file|
+          etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+
+          # Subtract a byte to ensure the range won't be empty.
+          # Avoids 416 (Range Not Satisfiable) responses.
+          response = @fetcher.call(remote_path, request_headers(etag, file.size - 1))
+          break true if response.is_a?(Gem::Net::HTTPNotModified)
+
+          file.digests = parse_digests(response)
+          # server may ignore Range and return the full response
+          if response.is_a?(Gem::Net::HTTPPartialContent)
+            tail = response.body.byteslice(1..-1)
+            break false unless tail && file.append(tail)
+          else
+            file.write(response.body)
+          end
+          CacheFile.write(etag_path, etag_from_response(response))
+          true
+        end
+      end
+
+      # request without range header to get the full file or a 304 Not Modified
+      def replace(remote_path, local_path, etag_path)
+        etag = etag_path.read.tap(&:chomp!) if etag_path.file?
+        response = @fetcher.call(remote_path, request_headers(etag))
+        return true if response.is_a?(Gem::Net::HTTPNotModified)
+        CacheFile.write(local_path, response.body, parse_digests(response))
+        CacheFile.write(etag_path, etag_from_response(response))
+      end
+
+      def request_headers(etag, range_start = nil)
+        headers = {}
+        headers["Range"] = "bytes=#{range_start}-" if range_start
+        headers["If-None-Match"] = %("#{etag}") if etag
+        headers
+      end
+
+      def etag_for_request(etag_path)
+        etag_path.read.tap(&:chomp!) if etag_path.file?
+      end
+
+      def etag_from_response(response)
+        return unless response["ETag"]
+        etag = response["ETag"].delete_prefix("W/")
+        return if etag.delete_prefix!('"') && !etag.delete_suffix!('"')
+        etag
+      end
+
+      # Unwraps and returns a Hash of digest algorithms and base64 values
+      # according to RFC 8941 Structured Field Values for HTTP.
+      # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence
+      # Ignores unsupported algorithms.
+      def parse_digests(response)
+        return unless header = response["Repr-Digest"] || response["Digest"]
+        digests = {}
+        header.split(",") do |param|
+          algorithm, value = param.split("=", 2)
+          algorithm.strip!
+          algorithm.downcase!
+          next unless SUPPORTED_DIGESTS.key?(algorithm)
+          next unless value = byte_sequence(value)
+          digests[algorithm] = value
+        end
+        digests.empty? ? nil : digests
+      end
+
+      # Unwrap surrounding colons (byte sequence)
+      # The wrapping characters must be matched or we return nil.
+      # Also handles quotes because right now rubygems.org sends them.
+      def byte_sequence(value)
+        return if value.delete_prefix!(":") && !value.delete_suffix!(":")
+        return if value.delete_prefix!('"') && !value.delete_suffix!('"')
+        value
+      end
+    end
+  end
+end

data/lib/bundler/compact_index_client.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Bundler
+  # The CompactIndexClient is responsible for fetching and parsing the compact index.
+  #
+  # The compact index is a set of caching optimized files that are used to fetch gem information.
+  # The files are:
+  # - names: a list of all gem names
+  # - versions: a list of all gem versions
+  # - info/[gem]: a list of all versions of a gem
+  #
+  # The client is instantiated with:
+  # - `directory`: the root directory where the cache files are stored.
+  # - `fetcher`: (optional) an object that responds to #call(uri_path, headers) and returns an http response.
+  # If the `fetcher` is not provided, the client will only read cached files from disk.
+  #
+  # The client is organized into:
+  # - `Updater`: updates the cached files on disk using the fetcher.
+  # - `Cache`: calls the updater, caches files, read and return them from disk
+  # - `Parser`: parses the compact index file data
+  # - `CacheFile`: a concurrency safe file reader/writer that verifies checksums
+  #
+  # The client is intended to optimize memory usage and performance.
+  # It is called 100s or 1000s of times, parsing files with hundreds of thousands of lines.
+  # It may be called concurrently without global interpreter lock in some Rubies.
+  # As a result, some methods may look more complex than necessary to save memory or time.
+  class CompactIndexClient
+    SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze
+    DEBUG_MUTEX = Thread::Mutex.new
+
+    # info returns an Array of INFO Arrays. Each INFO Array has the following indices:
+    INFO_NAME = 0
+    INFO_VERSION = 1
+    INFO_PLATFORM = 2
+    INFO_DEPS = 3
+    INFO_REQS = 4
+
+    def self.debug
+      return unless ENV["DEBUG_COMPACT_INDEX"]
+      DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
+    end
+
+    class Error < StandardError; end
+
+    require_relative "compact_index_client/cache"
+    require_relative "compact_index_client/cache_file"
+    require_relative "compact_index_client/parser"
+    require_relative "compact_index_client/updater"
+
+    def initialize(directory, fetcher = nil)
+      @cache = Cache.new(directory, fetcher)
+      @parser = Parser.new(@cache)
+    end
+
+    def names
+      Bundler::CompactIndexClient.debug { "names" }
+      @parser.names
+    end
+
+    def versions
+      Bundler::CompactIndexClient.debug { "versions" }
+      @parser.versions
+    end
+
+    def dependencies(names)
+      Bundler::CompactIndexClient.debug { "dependencies(#{names})" }
+      names.map {|name| info(name) }
+    end
+
+    def info(name)
+      Bundler::CompactIndexClient.debug { "info(#{name})" }
+      @parser.info(name)
+    end
+
+    def latest_version(name)
+      Bundler::CompactIndexClient.debug { "latest_version(#{name})" }
+      @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max
+    end
+
+    def available?
+      Bundler::CompactIndexClient.debug { "available?" }
+      @parser.available?
+    end
+
+    def reset!
+      Bundler::CompactIndexClient.debug { "reset!" }
+      @cache.reset!
+    end
+  end
+end

data/lib/bundler/constants.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+
+module Bundler
+  WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/
+  deprecate_constant :WINDOWS
+
+  FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd")
+  deprecate_constant :FREEBSD
+
+  NULL = File::NULL
+  deprecate_constant :NULL
+end

data/lib/bundler/current_ruby.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative "rubygems_ext"
+
+module Bundler
+  # Returns current version of Ruby
+  #
+  # @return [CurrentRuby] Current version of Ruby
+  def self.current_ruby
+    @current_ruby ||= CurrentRuby.new
+  end
+
+  class CurrentRuby
+    ALL_RUBY_VERSIONS = [*18..27, *30..34, *40..41].freeze
+    KNOWN_MINOR_VERSIONS = ALL_RUBY_VERSIONS.map {|v| v.digits.reverse.join(".") }.freeze
+    KNOWN_MAJOR_VERSIONS = ALL_RUBY_VERSIONS.map {|v| v.digits.last.to_s }.uniq.freeze
+    PLATFORM_MAP = {
+      ruby: [Gem::Platform::RUBY, CurrentRuby::ALL_RUBY_VERSIONS],
+      mri: [Gem::Platform::RUBY, CurrentRuby::ALL_RUBY_VERSIONS],
+      rbx: [Gem::Platform::RUBY],
+      truffleruby: [Gem::Platform::RUBY],
+      jruby: [Gem::Platform::JAVA, [18, 19]],
+      windows: [Gem::Platform::WINDOWS, CurrentRuby::ALL_RUBY_VERSIONS],
+      # deprecated
+      mswin: [Gem::Platform::MSWIN, CurrentRuby::ALL_RUBY_VERSIONS],
+      mswin64: [Gem::Platform::MSWIN64, CurrentRuby::ALL_RUBY_VERSIONS - [18]],
+      mingw: [Gem::Platform::UNIVERSAL_MINGW, CurrentRuby::ALL_RUBY_VERSIONS],
+      x64_mingw: [Gem::Platform::UNIVERSAL_MINGW, CurrentRuby::ALL_RUBY_VERSIONS - [18, 19]],
+    }.each_with_object({}) do |(platform, spec), hash|
+      hash[platform] = spec[0]
+      spec[1]&.each {|version| hash[:"#{platform}_#{version}"] = spec[0] }
+    end.freeze
+
+    def ruby?
+      return true if Bundler::MatchPlatform.generic_local_platform_is_ruby?
+
+      !windows? && (RUBY_ENGINE == "ruby" || RUBY_ENGINE == "rbx" || RUBY_ENGINE == "maglev" || RUBY_ENGINE == "truffleruby")
+    end
+
+    def mri?
+      !windows? && RUBY_ENGINE == "ruby"
+    end
+
+    def rbx?
+      ruby? && RUBY_ENGINE == "rbx"
+    end
+
+    def jruby?
+      RUBY_ENGINE == "jruby"
+    end
+
+    def maglev?
+      removed_message =
+        "`CurrentRuby#maglev?` was removed with no replacement. Please use the " \
+        "built-in Ruby `RUBY_ENGINE` constant to check the Ruby implementation you are running on."
+      SharedHelpers.feature_removed!(removed_message)
+    end
+
+    def truffleruby?
+      RUBY_ENGINE == "truffleruby"
+    end
+
+    def windows?
+      Gem.win_platform?
+    end
+    alias_method :mswin?, :windows?
+    alias_method :mswin64?, :windows?
+    alias_method :mingw?, :windows?
+    alias_method :x64_mingw?, :windows?
+
+    (KNOWN_MINOR_VERSIONS + KNOWN_MAJOR_VERSIONS).each do |version|
+      trimmed_version = version.tr(".", "")
+      define_method(:"on_#{trimmed_version}?") do
+        RUBY_VERSION.start_with?("#{version}.")
+      end
+
+      PLATFORM_MAP.keys.each do |platform|
+        define_method(:"#{platform}_#{trimmed_version}?") do
+          send(:"#{platform}?") && send(:"on_#{trimmed_version}?")
+        end
+      end
+
+      define_method(:"maglev_#{trimmed_version}?") do
+        removed_message =
+          "`CurrentRuby##{__method__}` was removed with no replacement. Please use the " \
+          "built-in Ruby `RUBY_ENGINE` and `RUBY_VERSION` constants to perform a similar check."
+
+        SharedHelpers.feature_removed!(removed_message)
+
+        send(:"maglev?") && send(:"on_#{trimmed_version}?")
+      end
+    end
+  end
+end