bundler 4.0.22 → 4.1.0.beta2

data/lib/bundler/vendor/thor/lib/thor.rb

@@ -625,7 +625,7 @@ class Bundler::Thor
     # alias name.
     def find_command_possibilities(meth)
       len = meth.to_s.length
-      possibilities = all_commands.reject { |_k, c| c.hidden? }.merge(map).keys.select { |n| meth == n[0, len] }.sort
+      possibilities = all_commands.reject {|k, v| v.is_a?(HiddenCommand) }.merge(map).keys.select { |n| meth == n[0, len] }.sort
       unique_possibilities = possibilities.map { |k| map[k] || k }.uniq
 
       if possibilities.include?(meth)
@@ -671,4 +671,37 @@ class Bundler::Thor
       self.class.help(shell, subcommand)
     end
   end
+
+  map TREE_MAPPINGS => :tree
+
+  desc "tree", "Print a tree of all available commands"
+  def tree
+    build_command_tree(self.class, "")
+  end
+
+private
+
+  def build_command_tree(klass, indent)
+    # Print current class name if it's not the root Bundler::Thor class
+    unless klass == Bundler::Thor
+      say "#{indent}#{klass.namespace || 'default'}", :blue
+      indent = "#{indent}  "
+    end
+
+    # Print all commands for this class
+    visible_commands = klass.commands.reject { |_, cmd| cmd.hidden? || cmd.name == "help" }
+    commands_count = visible_commands.count
+    visible_commands.sort.each_with_index do |(command_name, command), i|
+      description = command.description.split("\n").first || ""
+      icon = i == (commands_count - 1) ? "└─" : "├─"
+      say "#{indent}#{icon} ", nil, false
+      say command_name, :green, false
+      say " (#{description})" unless description.empty?
+    end
+
+    # Print all subcommands (from registered Bundler::Thor subclasses)
+    klass.subcommand_classes.each do |_, subclass|
+      build_command_tree(subclass, indent)
+    end
+  end
 end

data/lib/bundler/vendored_pub_grub.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
 
-module Bundler; end
-require_relative "vendor/pub_grub/lib/pub_grub"
+# Reuse RubyGems' vendored PubGrub (Gem::PubGrub). The Bundler gem ships a copy
+# under lib/rubygems/vendor, so this resolves even on RubyGems versions that
+# predate it.
+
+require "rubygems/vendor/pub_grub/lib/pub_grub"

data/lib/bundler/vendored_securerandom.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
-# Use RubyGems vendored copy when available. Otherwise fallback to Bundler
-# vendored copy. The vendored copy in Bundler can be removed once support for
-# RubyGems 3.5.18 is dropped.
+# Reuse RubyGems' vendored SecureRandom (Gem::SecureRandom). The Bundler gem
+# ships a copy under lib/rubygems/vendor, so this resolves even on RubyGems
+# versions that predate it. Fall back to the stdlib only when no vendored copy
+# is available at all.
 
-begin
-  require "rubygems/vendored_securerandom"
-rescue LoadError
-  require_relative "vendor/securerandom/lib/securerandom"
-  Gem::SecureRandom = Bundler::SecureRandom
+unless defined?(Gem::SecureRandom)
+  begin
+    require "rubygems/vendor/securerandom/lib/securerandom"
+  rescue LoadError
+    require "securerandom"
+    Gem::SecureRandom = SecureRandom
+  end
 end

data/lib/bundler/vendored_uri.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
-module Bundler; end
+# Reuse RubyGems' vendored URI (Gem::URI). The Bundler gem ships a copy under
+# lib/rubygems/vendor, so this resolves even on RubyGems versions that predate
+# it. Fall back to the stdlib only when no vendored copy is available at all.
 
-# Use RubyGems vendored copy when available. Otherwise fallback to Bundler
-# vendored copy. The vendored copy in Bundler can be removed once support for
-# RubyGems 3.5 is dropped.
+unless defined?(Gem::URI)
+  begin
+    require "rubygems/vendor/uri/lib/uri"
+  rescue LoadError
+    require "uri"
+    Gem::URI = URI
 
-begin
-  require "rubygems/vendor/uri/lib/uri"
-rescue LoadError
-  require_relative "vendor/uri/lib/uri"
-  Gem::URI = Bundler::URI
-
-  module Gem
-    def URI(uri) # rubocop:disable Naming/MethodName
-      Bundler::URI(uri)
+    module Gem
+      def URI(uri) # rubocop:disable Naming/MethodName
+        Kernel.URI(uri)
+      end
+      module_function :URI
     end
-    module_function :URI
   end
 end

data/lib/bundler/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: false
 
 module Bundler
-  VERSION = "4.0.22".freeze
+  VERSION = "4.1.0.beta2".freeze
 
   def self.bundler_major_version
     @bundler_major_version ||= gem_version.segments.first

data/lib/bundler/worker.rb

@@ -19,11 +19,12 @@ module Bundler
     # @param size [Integer] Size of pool
     # @param name [String] name the name of the worker
     # @param func [Proc] job to run in inside the worker pool
-    def initialize(size, name, func)
+    # @param response_queue [Thread::Queue] queue that receives completed jobs
+    def initialize(size, name, func, response_queue: Thread::Queue.new)
       @name = name
       @request_queue = Thread::Queue.new
       @request_queue_with_priority = Thread::Queue.new
-      @response_queue = Thread::Queue.new
+      @response_queue = response_queue
       @func = func
       @size = size
       @threads = nil

data/lib/bundler.rb

@@ -41,7 +41,6 @@ module Bundler
   autoload :Checksum,               File.expand_path("bundler/checksum", __dir__)
   autoload :CLI,                    File.expand_path("bundler/cli", __dir__)
   autoload :CIDetector,             File.expand_path("bundler/ci_detector", __dir__)
-  autoload :CompactIndexClient,     File.expand_path("bundler/compact_index_client", __dir__)
   autoload :Definition,             File.expand_path("bundler/definition", __dir__)
   autoload :Dependency,             File.expand_path("bundler/dependency", __dir__)
   autoload :Deprecate,              File.expand_path("bundler/deprecate", __dir__)
@@ -62,6 +61,7 @@ module Bundler
   autoload :MatchRemoteMetadata,    File.expand_path("bundler/match_remote_metadata", __dir__)
   autoload :Materialization,        File.expand_path("bundler/materialization", __dir__)
   autoload :NULL,                   File.expand_path("bundler/constants", __dir__)
+  autoload :Override,               File.expand_path("bundler/override", __dir__)
   autoload :ProcessLock,            File.expand_path("bundler/process_lock", __dir__)
   autoload :RemoteSpecification,    File.expand_path("bundler/remote_specification", __dir__)
   autoload :Resolver,               File.expand_path("bundler/resolver", __dir__)

data/lib/rubygems/compact_index_client/cache.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "pathname" unless defined?(Pathname)
+require "set"
+
+class Gem::CompactIndexClient
+  # Calls the Updater to update the cached files on disk, reads the
+  # cached files and returns their contents.
+  class Cache
+    attr_reader :directory
+
+    def initialize(directory, fetcher = nil)
+      @directory = Pathname.new(directory).expand_path
+      @updater = Updater.new(fetcher) if fetcher
+      @mutex = Thread::Mutex.new
+      @endpoints = Set.new
+
+      @info_root = mkdir("info")
+      @special_characters_info_root = mkdir("info-special-characters")
+      @info_etag_root = mkdir("info-etags")
+    end
+
+    def names
+      fetch("names", names_path, names_etag_path)
+    end
+
+    def versions
+      fetch("versions", versions_path, versions_etag_path)
+    end
+
+    def info(name, remote_checksum = nil)
+      path = info_path(name)
+
+      if remote_checksum && remote_checksum != checksum_for_file(path)
+        fetch("info/#{name}", path, info_etag_path(name))
+      else
+        Gem::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum matches local" : "versions index checksum is nil"})" }
+        read(path)
+      end
+    end
+
+    # Fetch a single gem's info file without consulting the versions
+    # index, refreshing the cached file with a conditional request.
+    def fetch_info(name)
+      fetch("info/#{name}", info_path(name), info_etag_path(name))
+    end
+
+    def reset!
+      @mutex.synchronize { @endpoints.clear }
+    end
+
+    private
+
+    def names_path = directory.join("names")
+    def names_etag_path = directory.join("names.etag")
+    def versions_path = directory.join("versions")
+    def versions_etag_path = directory.join("versions.etag")
+
+    def info_path(name)
+      name = name.to_s
+      validate_name!(name)
+      if /[^a-z0-9_-]/.match?(name)
+        name += "-#{Digest::MD5.hexdigest(name).downcase}"
+        @special_characters_info_root.join(name)
+      else
+        @info_root.join(name)
+      end
+    end
+
+    def info_etag_path(name)
+      name = name.to_s
+      validate_name!(name)
+      @info_etag_root.join("#{name}-#{Digest::MD5.hexdigest(name).downcase}")
+    end
+
+    # Gem names come from the remote index and are not otherwise
+    # validated, so refuse anything that would escape the cache
+    # directory when used as a path component.
+    def validate_name!(name)
+      return unless name.empty? || name.include?("\0") || name == "." || name == ".." || File.basename(name) != name
+
+      raise Gem::Exception, "malformed gem name: #{name.inspect}"
+    end
+
+    def checksum_for_file(path)
+      return unless path.file?
+      Gem::CompactIndexClient.filesystem_access(path, :read) do
+        Digest::MD5.file(path).hexdigest
+      end
+    end
+
+    def mkdir(name)
+      directory.join(name).tap do |dir|
+        Gem::CompactIndexClient.filesystem_access(dir) do
+          FileUtils.mkdir_p(dir)
+        end
+      end
+    end
+
+    def fetch(remote_path, path, etag_path)
+      if already_fetched?(remote_path)
+        Gem::CompactIndexClient.debug { "already fetched #{remote_path}" }
+      else
+        Gem::CompactIndexClient.debug { "fetching #{remote_path}" }
+        @updater&.update(remote_path, path, etag_path)
+      end
+
+      read(path)
+    end
+
+    def already_fetched?(remote_path)
+      @mutex.synchronize { !@endpoints.add?(remote_path) }
+    end
+
+    def read(path)
+      return unless path.file?
+      Gem::CompactIndexClient.filesystem_access(path, :read, &:read)
+    end
+  end
+end

data/lib/rubygems/compact_index_client/cache_file.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "rubygems/package"
+
+class Gem::CompactIndexClient
+  # write cache files in a way that is robust to concurrent modifications
+  # if digests are given, the checksums will be verified
+  class CacheFile
+    DEFAULT_FILE_MODE = 0o644
+    private_constant :DEFAULT_FILE_MODE
+
+    class Error < RuntimeError; end
+    class ClosedError < Error; end
+
+    class DigestMismatchError < Error
+      def initialize(digests, expected_digests)
+        super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}."
+      end
+    end
+
+    # Initialize with a copy of the original file, then yield the instance.
+    def self.copy(path, &block)
+      new(path) do |file|
+        file.initialize_digests
+
+        Gem::CompactIndexClient.filesystem_access(path, :read) do
+          path.open("rb") do |s|
+            file.open {|f| IO.copy_stream(s, f) }
+          end
+        end
+
+        yield file
+      end
+    end
+
+    # Write data to a temp file, then replace the original file with it verifying the digests if given.
+    def self.write(path, data, digests = nil)
+      return unless data
+      new(path) do |file|
+        file.digests = digests
+        file.write(data)
+      end
+    end
+
+    attr_reader :original_path, :path
+
+    def initialize(original_path, &block)
+      @original_path = original_path
+      @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE
+      @path = original_path.sub(/$/, ".#{$$}.tmp")
+      return unless block_given?
+      begin
+        yield self
+      ensure
+        close
+      end
+    end
+
+    def size
+      path.size
+    end
+
+    # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys.
+    def initialize_digests(keys = nil)
+      @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup
+      @digests.transform_values! {|algo_class| Digest(algo_class).new }
+    end
+
+    # reset the digests so they don't contain any previously read data
+    def reset_digests
+      @digests&.each_value(&:reset)
+    end
+
+    # set the digests that will be verified at the end
+    def digests=(expected_digests)
+      @expected_digests = expected_digests
+
+      if @expected_digests.nil?
+        @digests = nil
+      elsif @digests
+        @digests = @digests.slice(*@expected_digests.keys)
+      else
+        initialize_digests(@expected_digests.keys)
+      end
+    end
+
+    def digests?
+      @digests&.any?
+    end
+
+    # Open the temp file for writing, reusing original permissions, yielding the IO object.
+    def open(write_mode = "wb", perm = @perm, &block)
+      raise ClosedError, "Cannot reopen closed file" if @closed
+      Gem::CompactIndexClient.filesystem_access(path, :write) do
+        path.open(write_mode, perm) do |f|
+          yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f
+        end
+      end
+    end
+
+    # Returns false without appending when no digests since appending is too error prone to do without digests.
+    def append(data)
+      return false unless digests?
+      open("ab") {|f| f.write data }
+      verify && commit
+    end
+
+    def write(data)
+      reset_digests
+      open {|f| f.write data }
+      commit!
+    end
+
+    def commit!
+      verify || raise(DigestMismatchError.new(@base64digests, @expected_digests))
+      commit
+    end
+
+    # Verify the digests, returning true on match, false on mismatch.
+    def verify
+      return true unless @expected_digests && digests?
+      @base64digests = @digests.transform_values!(&:base64digest)
+      @digests = nil
+      @base64digests.all? {|algo, digest| @expected_digests[algo] == digest }
+    end
+
+    # Replace the original file with the temp file without verifying digests.
+    # The file is permanently closed.
+    def commit
+      raise ClosedError, "Cannot commit closed file" if @closed
+      Gem::CompactIndexClient.filesystem_access(original_path, :write) do
+        FileUtils.mv(path, original_path)
+      end
+      @closed = true
+    end
+
+    # Remove the temp file without replacing the original file.
+    # The file is permanently closed.
+    def close
+      return if @closed
+      FileUtils.remove_file(path) if @path&.file?
+      @closed = true
+    end
+  end
+end

data/lib/rubygems/compact_index_client/http_fetcher.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "rubygems/remote_fetcher"
+
+class Gem::CompactIndexClient
+  # Fetches compact index files relative to +base_uri+ using
+  # Gem::RemoteFetcher's connection infrastructure (proxy, TLS,
+  # connection pooling). Implements the fetcher interface expected by
+  # Gem::CompactIndexClient: #call(path, headers) returning a
+  # Gem::Net::HTTP response.
+  class HTTPFetcher
+    REDIRECT_LIMIT = 10
+    private_constant :REDIRECT_LIMIT
+
+    def initialize(base_uri, remote_fetcher = Gem::RemoteFetcher.fetcher)
+      base_uri = base_uri.to_s
+      base_uri += "/" unless base_uri.end_with?("/")
+      @base_uri = Gem::URI(base_uri)
+      @remote_fetcher = remote_fetcher
+    end
+
+    def call(path, headers = {})
+      fetch(@base_uri + path, headers, REDIRECT_LIMIT)
+    end
+
+    private
+
+    def fetch(uri, headers, redirects_remaining)
+      response = request(uri, headers)
+
+      case response
+      when Gem::Net::HTTPNotModified
+        response
+      when Gem::Net::HTTPSuccess
+        # The callers write the body into the cache, so a body-less success
+        # such as 204 would truncate the cached file.
+        raise bad_response(response, uri) unless response.class.body_permitted?
+
+        response
+      when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther,
+           Gem::Net::HTTPTemporaryRedirect, Gem::Net::HTTPPermanentRedirect
+        raise Gem::RemoteFetcher::FetchError.new("too many redirects", uri) if redirects_remaining.zero?
+
+        location = response["Location"]
+        raise Gem::RemoteFetcher::FetchError.new("redirecting but no redirect location was given", uri) unless location
+
+        redirect = uri + location
+        if https?(uri) && !https?(redirect)
+          raise Gem::RemoteFetcher::FetchError.new("redirecting to non-https resource: #{Gem::Uri.redact(redirect)}", uri)
+        end
+        # An absolute Location on the same origin drops the credentials that a
+        # relative one would have kept. Another port or scheme on the same host
+        # is a different protection space (RFC 9110, Section 11.5).
+        same_origin = [redirect.scheme, redirect.host, redirect.port] == [uri.scheme, uri.host, uri.port]
+        redirect.userinfo = uri.userinfo if same_origin && !redirect.userinfo
+
+        fetch(redirect, headers, redirects_remaining - 1)
+      when Gem::Net::HTTPRangeNotSatisfiable
+        raise bad_response(response, uri) unless headers.key?("Range")
+
+        # The local cache is longer than the remote file, refetch it whole. A
+        # matching ETag would otherwise turn the retry into a 304 and keep the
+        # oversized cache.
+        fetch(uri, headers.except("Range", "If-None-Match"), redirects_remaining)
+      else
+        raise bad_response(response, uri)
+      end
+    end
+
+    # The callers fall back to the Marshal index when a fetch fails, and they
+    # only recognize a failure that arrives as a FetchError.
+    def request(uri, headers)
+      @remote_fetcher.request(uri, Gem::Net::HTTP::Get) do |req|
+        headers.each {|name, value| req[name] = value }
+      end
+    rescue Gem::Timeout::Error, IOError, SocketError, SystemCallError,
+           *(OpenSSL::SSL::SSLError if Gem::HAVE_OPENSSL) => e
+      raise Gem::RemoteFetcher::FetchError.new("#{e.class}: #{e}", uri)
+    end
+
+    def bad_response(response, uri)
+      detail = response["X-Error-Message"] || response.message
+      Gem::RemoteFetcher::FetchError.new("bad response #{detail} #{response.code}", uri)
+    end
+
+    def https?(uri)
+      uri.scheme == "https"
+    end
+  end
+end

data/lib/rubygems/compact_index_client/parser.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "rubygems/resolver/api_set/gem_parser"
+
+class Gem::CompactIndexClient
+  class Parser
+    # `compact_index` - an object responding to #names, #versions, #info(name, checksum),
+    #                   returning the file contents as a string
+    def initialize(compact_index)
+      @compact_index = compact_index
+      @info_checksums = nil
+      @versions_by_name = nil
+      @available = nil
+      @gem_parser = nil
+    end
+
+    def names
+      lines(@compact_index.names)
+    end
+
+    def versions
+      @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] }
+      @info_checksums = {}
+
+      lines(@compact_index.versions).each do |line|
+        name, versions_string, checksum = line.split(" ", 3)
+        @info_checksums[name] = checksum || ""
+        versions_string.split(",") do |version|
+          delete = version.delete_prefix!("-")
+          version = version.split("-", 2).unshift(name)
+          if delete
+            @versions_by_name[name].delete(version)
+          else
+            @versions_by_name[name] << version
+          end
+        end
+      end
+
+      @versions_by_name
+    end
+
+    def info(name)
+      parse_info(@compact_index.info(name, info_checksums[name]), name)
+    end
+
+    def parse_info(data, name)
+      lines(data).map {|line| gem_parser.parse(line).unshift(name) }
+    end
+
+    def available?
+      return @available unless @available.nil?
+      @available = !info_checksums.empty?
+    end
+
+    private
+
+    def info_checksums
+      @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums|
+        parse_version_checksum(line, checksums)
+      end
+    end
+
+    def lines(data)
+      return [] if data.nil? || data.empty?
+      lines = data.split("\n")
+      header = lines.index("---")
+      header ? lines[header + 1..-1] : lines
+    end
+
+    def gem_parser
+      @gem_parser ||= Gem::Resolver::APISet::GemParser.new
+    end
+
+    # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects.
+    # This method gets called at least once for every gem when parsing versions.
+    def parse_version_checksum(line, checksums)
+      return unless (name_end = line.index(" ")) # Artifactory bug causes blank lines in artifactory index files
+      checksum_start = line.index(" ", name_end + 1)
+      return unless checksum_start
+      checksum_start += 1
+
+      checksum_end = line.size - checksum_start
+
+      line.freeze # allows slicing into the string to not allocate a copy of the line
+      name = line[0, name_end]
+      checksum = line[checksum_start, checksum_end]
+      checksums[name.freeze] = checksum # freeze name since it is used as a hash key
+    end
+  end
+end