bundler-clone 4.0.15

data/lib/bundler/installer/parallel_installer.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require_relative "../worker"
+require_relative "gem_installer"
+
+module Bundler
+  class ParallelInstaller
+    class SpecInstallation
+      attr_accessor :spec, :name, :full_name, :post_install_message, :state, :error, :dependencies
+      def initialize(spec)
+        @spec = spec
+        @name = spec.name
+        @full_name = spec.full_name
+        @state = :none
+        @post_install_message = ""
+        @error = nil
+      end
+
+      def installed?
+        state == :installed
+      end
+
+      def enqueued?
+        state == :enqueued
+      end
+
+      def enqueue_with_priority?
+        state == :installable && spec.extensions.any?
+      end
+
+      def failed?
+        state == :failed
+      end
+
+      def ready_to_enqueue?
+        state == :none
+      end
+
+      def ready_to_install?(installed_specs)
+        return false unless state == :downloaded
+
+        spec.extensions.none? || dependencies_installed?(installed_specs)
+      end
+
+      def has_post_install_message?
+        !post_install_message.empty?
+      end
+
+      # Recursively checks that all dependencies (direct and transitive) have been installed.
+      def dependencies_installed?(installed_specs)
+        dependencies.all? do |dep|
+          installed_specs.include?(dep.name) && dep.dependencies_installed?(installed_specs)
+        end
+      end
+
+      def to_s
+        "#<#{self.class} #{full_name} (#{state})>"
+      end
+    end
+
+    def self.call(*args, **kwargs)
+      new(*args, **kwargs).call
+    end
+
+    attr_reader :size
+
+    def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil)
+      @installer = installer
+      @size = size
+      @standalone = standalone
+      @force = force
+      @local = local
+      @specs = all_specs.map {|s| SpecInstallation.new(s) }
+      specs_by_name = @specs.to_h {|s| [s.name, s] }
+      @specs.each do |spec_install|
+        spec_install.dependencies = spec_install.spec.dependencies.filter_map do |dep|
+          specs_by_name[dep.name] unless dep.type == :development || dep.name == spec_install.name
+        end
+      end
+      @specs.each do |spec_install|
+        spec_install.state = :installed if skip.include?(spec_install.name)
+      end if skip
+      @spec_set = all_specs
+      @rake = @specs.find {|s| s.name == "rake" unless s.installed? }
+    end
+
+    def call
+      if @rake
+        do_download(@rake, 0)
+        do_install(@rake, 0)
+        Gem::Specification.reset
+      end
+
+      if @size > 1
+        install_with_worker
+      else
+        install_serially
+      end
+
+      handle_error if failed_specs.any?
+      @specs
+    ensure
+      worker_pool&.stop
+    end
+
+    private
+
+    def failed_specs
+      @specs.select(&:failed?)
+    end
+
+    def install_with_worker
+      with_jobserver do
+        installed_specs = {}
+        enqueue_specs(installed_specs)
+
+        process_specs(installed_specs) until finished_installing?
+      end
+    end
+
+    def with_jobserver
+      # The jobserver hands tokens to child `make` processes through MAKEFLAGS
+      # using the GNU make `--jobserver-auth` protocol. nmake, the default make
+      # on mswin, instead reads MAKEFLAGS as bare option letters and aborts
+      # every native extension build with `fatal error U1065: invalid option
+      # '-'`. Skip the jobserver when nmake is in use. Other Windows toolchains
+      # such as mingw use GNU make and keep working through the inherited pipe.
+      return yield if nmake?
+
+      begin
+        r, w = IO.pipe
+        r.close_on_exec = false
+        w.close_on_exec = false
+        w.write("*" * @size)
+
+        old_makeflags = ENV["MAKEFLAGS"]
+        ENV["MAKEFLAGS"] = [old_makeflags, "--jobserver-auth=#{r.fileno},#{w.fileno}"].compact.join(" ")
+
+        yield
+      ensure
+        # Restore MAKEFLAGS before closing the pipe so a close failure can't
+        # leave the process with descriptors that point at a closed pipe.
+        old_makeflags ? ENV["MAKEFLAGS"] = old_makeflags : ENV.delete("MAKEFLAGS")
+
+        r&.close
+        w&.close
+      end
+    end
+
+    # Mirror how RubyGems' extension builder picks the make program so the
+    # jobserver is only set up when a GNU-compatible make will consume it.
+    def nmake?
+      make = ENV["MAKE"] || ENV["make"]
+      make ||= "nmake" if RUBY_PLATFORM.include?("mswin")
+      /\bnmake/i.match?(make.to_s)
+    end
+
+    def install_serially
+      until finished_installing?
+        raise "failed to find a spec to enqueue while installing serially" unless spec_install = @specs.find(&:ready_to_enqueue?)
+        spec_install.state = :enqueued
+        do_download(spec_install, 0)
+        do_install(spec_install, 0)
+      end
+    end
+
+    def worker_pool
+      @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num|
+        case spec_install.state
+        when :enqueued
+          do_download(spec_install, worker_num)
+        when :installable
+          do_install(spec_install, worker_num)
+        else
+          spec_install
+        end
+      }
+    end
+
+    def do_download(spec_install, worker_num)
+      Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL, spec_install)
+
+      gem_installer = Bundler::GemInstaller.new(
+        spec_install.spec, @installer, @standalone, worker_num, @force, @local
+      )
+
+      success, message = gem_installer.download
+
+      if success
+        spec_install.state = :downloaded
+      else
+        spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}"
+        spec_install.state = :failed
+      end
+
+      spec_install
+    end
+
+    def do_install(spec_install, worker_num)
+      gem_installer = Bundler::GemInstaller.new(
+        spec_install.spec, @installer, @standalone, worker_num, @force, @local
+      )
+      success, message = gem_installer.install_from_spec
+      if success
+        spec_install.state = :installed
+        spec_install.post_install_message = message unless message.nil?
+      else
+        spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}"
+        spec_install.state = :failed
+      end
+      Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install)
+      spec_install
+    end
+
+    # Dequeue a spec and save its post-install message and then enqueue the
+    # remaining specs.
+    # Some specs might've had to wait til this spec was installed to be
+    # processed so the call to `enqueue_specs` is important after every
+    # dequeue.
+    def process_specs(installed_specs)
+      spec = worker_pool.deq
+
+      if spec.installed?
+        installed_specs[spec.name] = true
+        return
+      elsif spec.failed?
+        return
+      elsif spec.ready_to_install?(installed_specs)
+        spec.state = :installable
+      end
+
+      worker_pool.enq(spec, priority: spec.enqueue_with_priority?)
+    end
+
+    def finished_installing?
+      @specs.all? do |spec|
+        return true if spec.failed?
+        spec.installed?
+      end
+    end
+
+    def handle_error
+      errors = failed_specs.map(&:error)
+      if exception = errors.find {|e| e.is_a?(Bundler::BundlerError) }
+        raise exception
+      end
+      raise Bundler::InstallError, errors.join("\n\n")
+    end
+
+    def require_tree_for_spec(spec)
+      tree = @spec_set.what_required(spec)
+      t = String.new("In #{File.basename(SharedHelpers.default_gemfile)}:\n")
+      tree.each_with_index do |s, depth|
+        t << "  " * depth.succ << s.name
+        unless tree.last == s
+          t << %( was resolved to #{s.version}, which depends on)
+        end
+        t << %(\n)
+      end
+      t
+    end
+
+    # Keys in the remains hash represent uninstalled gems specs.
+    # We enqueue all gem specs that do not have any dependencies.
+    # Later we call this lambda again to install specs that depended on
+    # previously installed specifications. We continue until all specs
+    # are installed.
+    def enqueue_specs(installed_specs)
+      @specs.each do |spec|
+        if spec.installed?
+          installed_specs[spec.name] = true
+          next
+        end
+
+        spec.state = :enqueued
+        worker_pool.enq spec
+      end
+    end
+  end
+end

data/lib/bundler/installer/standalone.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Standalone
+    def initialize(groups, definition)
+      @specs = definition.specs_for(groups)
+    end
+
+    def generate
+      SharedHelpers.filesystem_access(bundler_path) do |p|
+        FileUtils.mkdir_p(p)
+      end
+      File.open File.join(bundler_path, "setup.rb"), "w" do |file|
+        file.puts "require 'rbconfig'"
+        file.puts prevent_gem_activation
+        file.puts define_path_helpers
+        file.puts reverse_rubygems_kernel_mixin
+        paths.each do |path|
+          if Pathname.new(path).absolute?
+            file.puts %($:.unshift "#{path}")
+          else
+            file.puts %($:.unshift File.expand_path("\#{__dir__}/#{path}"))
+          end
+        end
+      end
+    end
+
+    private
+
+    def paths
+      @specs.flat_map do |spec|
+        next if spec.name == "bundler"
+        Array(spec.require_paths).map do |path|
+          gem_path(path, spec).
+            sub(version_dir, '#{RUBY_ENGINE}/#{Gem.ruby_api_version}').
+            sub(extensions_dir, 'extensions/\k<platform>/#{Gem.extension_api_version}')
+          # This is a static string intentionally. It's interpolated at a later time.
+        end
+      end.compact
+    end
+
+    def version_dir
+      "#{RUBY_ENGINE}/#{Gem.ruby_api_version}"
+    end
+
+    def extensions_dir
+      %r{extensions/(?<platform>[^/]+)/#{Regexp.escape(Gem.extension_api_version)}}
+    end
+
+    def bundler_path
+      Bundler.root.join(Bundler.settings[:path].to_s, "bundler")
+    end
+
+    def gem_path(path, spec)
+      full_path = Pathname.new(path).absolute? ? path : File.join(spec.full_gem_path, path)
+      if spec.source.instance_of?(Source::Path) && spec.source.path.absolute?
+        full_path
+      else
+        SharedHelpers.relative_path_to(full_path, from: Bundler.root.join(bundler_path))
+      end
+    end
+
+    def prevent_gem_activation
+      <<~'END'
+        module Kernel
+          remove_method(:gem) if private_method_defined?(:gem)
+
+          def gem(*)
+          end
+
+          private :gem
+        end
+      END
+    end
+
+    def define_path_helpers
+      <<~'END'
+        unless defined?(Gem)
+          module Gem
+            def self.ruby_api_version
+              RbConfig::CONFIG["ruby_version"]
+            end
+
+            def self.extension_api_version
+              if 'no' == RbConfig::CONFIG['ENABLE_SHARED']
+                "#{ruby_api_version}-static"
+              else
+                ruby_api_version
+              end
+            end
+          end
+        end
+      END
+    end
+
+    def reverse_rubygems_kernel_mixin
+      <<~END
+      if Gem.respond_to?(:discover_gems_on_require=)
+        Gem.discover_gems_on_require = false
+      else
+        [::Kernel.singleton_class, ::Kernel].each do |k|
+          if k.private_method_defined?(:gem_original_require)
+            private_require = k.private_method_defined?(:require)
+            k.send(:remove_method, :require)
+            k.send(:define_method, :require, k.instance_method(:gem_original_require))
+            k.send(:private, :require) if private_require
+          end
+        end
+      end
+      END
+    end
+  end
+end

data/lib/bundler/installer.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require_relative "worker"
+require_relative "installer/parallel_installer"
+require_relative "installer/standalone"
+require_relative "installer/gem_installer"
+
+module Bundler
+  class Installer
+    attr_reader :post_install_messages, :definition
+
+    # Begins the installation process for Bundler.
+    # For more information see the #run method on this class.
+    def self.install(root, definition, options = {})
+      installer = new(root, definition)
+      Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL_ALL, definition.dependencies)
+      installer.run(options)
+      Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL_ALL, definition.dependencies)
+      installer
+    end
+
+    def initialize(root, definition)
+      @root = root
+      @definition = definition
+      @post_install_messages = {}
+    end
+
+    # Runs the install procedures for a specific Gemfile.
+    #
+    # Firstly, this method will check to see if `Bundler.bundle_path` exists
+    # and if not then Bundler will create the directory. This is usually the same
+    # location as RubyGems which typically is the `~/.gem` directory
+    # unless other specified.
+    #
+    # Secondly, it checks if Bundler has been configured to be "frozen".
+    # Frozen ensures that the Gemfile and the Gemfile.lock file are matching.
+    # This stops a situation where a developer may update the Gemfile but may not run
+    # `bundle install`, which leads to the Gemfile.lock file not being correctly updated.
+    # If this file is not correctly updated then any other developer running
+    # `bundle install` will potentially not install the correct gems.
+    #
+    # Thirdly, Bundler checks if there are any dependencies specified in the Gemfile.
+    # If there are no dependencies specified then Bundler returns a warning message stating
+    # so and this method returns.
+    #
+    # Fourthly, Bundler checks if the Gemfile.lock exists, and if so
+    # then proceeds to set up a definition based on the Gemfile and the Gemfile.lock.
+    # During this step Bundler will also download information about any new gems
+    # that are not in the Gemfile.lock and resolve any dependencies if needed.
+    #
+    # Fifthly, Bundler resolves the dependencies either through a cache of gems or by remote.
+    # This then leads into the gems being installed, along with stubs for their executables,
+    # but only if the --binstubs option has been passed or Bundler.options[:bin] has been set
+    # earlier.
+    #
+    # Sixthly, a new Gemfile.lock is created from the installed gems to ensure that the next time
+    # that a user runs `bundle install` they will receive any updates from this process.
+    #
+    # Finally, if the user has specified the standalone flag, Bundler will generate the needed
+    # require paths and save them in a `setup.rb` file. See `bundle standalone --help` for more
+    # information.
+    def run(options)
+      Bundler.create_bundle_path
+
+      ProcessLock.lock do
+        # Invalidate any stale gem specification cache from before we acquired the lock.
+        # Another process may have installed gems while we were waiting.
+        Gem::Specification.reset
+        @definition.sources.clear_cache
+
+        @definition.ensure_equivalent_gemfile_and_lockfile(options[:deployment])
+
+        if @definition.dependencies.empty?
+          Bundler.ui.warn "The Gemfile specifies no dependencies"
+          lock
+          return
+        end
+
+        if @definition.setup_domain!(options)
+          ensure_specs_are_compatible!
+          load_plugins
+        end
+        install(options)
+
+        Gem::Specification.reset # invalidate gem specification cache so that installed gems are immediately available
+
+        lock
+        Standalone.new(options[:standalone], @definition).generate if options[:standalone]
+      end
+    end
+
+    def generate_bundler_executable_stubs(spec, options = {})
+      if spec.name == "bundler"
+        Bundler.ui.warn "Bundler itself does not use binstubs because its version is selected by RubyGems"
+        return
+      end
+
+      if options[:binstubs_cmd] && spec.executables.empty?
+        options = {}
+        spec.runtime_dependencies.each do |dep|
+          bins = @definition.specs[dep].first.executables
+          options[dep.name] = bins unless bins.empty?
+        end
+        if options.any?
+          Bundler.ui.warn "#{spec.name} has no executables, but you may want " \
+            "one from a gem it depends on."
+          options.each {|name, bins| Bundler.ui.warn "  #{name} has: #{bins.join(", ")}" }
+        else
+          Bundler.ui.warn "There are no executables for the gem #{spec.name}."
+        end
+        return
+      end
+
+      # double-assignment to avoid warnings about variables that will be used by ERB
+      bin_path = Bundler.bin_path
+      bin_path = bin_path
+      relative_gemfile_path = Bundler.default_gemfile.relative_path_from(bin_path)
+      relative_gemfile_path = relative_gemfile_path
+      ruby_command = Thor::Util.ruby_command
+      ruby_command = ruby_command
+      template_path = File.expand_path("templates/Executable", __dir__)
+      template = File.read(template_path)
+
+      exists = []
+      spec.executables.each do |executable|
+        binstub_path = "#{bin_path}/#{executable}"
+        if File.exist?(binstub_path) && !options[:force]
+          exists << executable
+          next
+        end
+
+        mode = Gem.win_platform? ? "wb:UTF-8" : "w"
+        require "erb"
+        content = ERB.new(template, trim_mode: "-").result(binding)
+
+        File.write(binstub_path, content, mode: mode, perm: 0o777 & ~File.umask)
+        if Gem.win_platform? || options[:all_platforms]
+          prefix = "@ruby -x \"%~f0\" %*\n@exit /b %ERRORLEVEL%\n\n"
+          File.write("#{binstub_path}.cmd", prefix + content, mode: mode)
+        end
+      end
+
+      if options[:binstubs_cmd] && exists.any?
+        case exists.size
+        when 1
+          Bundler.ui.warn "Skipped #{exists[0]} since it already exists."
+        when 2
+          Bundler.ui.warn "Skipped #{exists.join(" and ")} since they already exist."
+        else
+          items = exists[0...-1].empty? ? nil : exists[0...-1].join(", ")
+          skipped = [items, exists[-1]].compact.join(" and ")
+          Bundler.ui.warn "Skipped #{skipped} since they already exist."
+        end
+        Bundler.ui.warn "If you want to overwrite skipped stubs, use --force."
+      end
+    end
+
+    def generate_standalone_bundler_executable_stubs(spec, options = {})
+      # double-assignment to avoid warnings about variables that will be used by ERB
+      bin_path = Bundler.bin_path
+      unless path = Bundler.settings[:path]
+        raise "Can't standalone without an explicit path set"
+      end
+      standalone_path = Bundler.root.join(path).relative_path_from(bin_path)
+      standalone_path = standalone_path
+      template = File.read(File.expand_path("templates/Executable.standalone", __dir__))
+      ruby_command = Thor::Util.ruby_command
+      ruby_command = ruby_command
+
+      spec.executables.each do |executable|
+        next if executable == "bundle"
+        executable_path = Pathname(spec.full_gem_path).join(spec.bindir, executable).relative_path_from(bin_path)
+        executable_path = executable_path
+
+        mode = Gem.win_platform? ? "wb:UTF-8" : "w"
+        require "erb"
+        content = ERB.new(template, trim_mode: "-").result(binding)
+
+        File.write("#{bin_path}/#{executable}", content, mode: mode, perm: 0o755)
+        if Gem.win_platform? || options[:all_platforms]
+          prefix = "@ruby -x \"%~f0\" %*\n@exit /b %ERRORLEVEL%\n\n"
+          File.write("#{bin_path}/#{executable}.cmd", prefix + content, mode: mode)
+        end
+      end
+    end
+
+    private
+
+    # the order that the resolver provides is significant, since
+    # dependencies might affect the installation of a gem.
+    # that said, it's a rare situation (other than rake), and parallel
+    # installation is SO MUCH FASTER. so we let people opt in.
+    def install(options)
+      standalone = options[:standalone]
+      force = options[:force]
+      local = options[:local] || options[:"prefer-local"]
+      jobs = Bundler.settings.installation_parallelization
+      specs = @definition.specs
+      # Installing default gems may need the remote index again to cache
+      # their .gem files, so keep resolution memory around in that case.
+      # The bundler spec itself is excluded because it comes from the
+      # metadata source and never goes through that path.
+      @definition.release_resolution_memory! if specs.none? {|s| s.default_gem? && s.source.is_a?(Source::Rubygems) }
+      spec_installations = ParallelInstaller.call(self, specs, jobs, standalone, force, local: local)
+      spec_installations.each do |installation|
+        post_install_messages[installation.name] = installation.post_install_message if installation.has_post_install_message?
+      end
+    end
+
+    def load_plugins
+      Gem.load_plugins
+
+      requested_path_gems = @definition.specs.select {|s| s.source.is_a?(Source::Path) }
+      path_plugin_files = requested_path_gems.flat_map do |spec|
+        spec.matches_for_glob("rubygems_plugin#{Bundler.rubygems.suffix_pattern}")
+      rescue TypeError
+        error_message = "#{spec.name} #{spec.version} has an invalid gemspec"
+        raise Gem::InvalidSpecificationException, error_message
+      end
+      Gem.load_plugin_files(path_plugin_files)
+      Gem.load_env_plugins
+    end
+
+    def ensure_specs_are_compatible!
+      @definition.specs.each do |spec|
+        unless spec.matches_current_ruby?
+          raise InstallError, "#{spec.full_name} requires ruby version #{spec.required_ruby_version}, " \
+            "which is incompatible with the current version, #{Gem.ruby_version}"
+        end
+        unless spec.matches_current_rubygems?
+          raise InstallError, "#{spec.full_name} requires rubygems version #{spec.required_rubygems_version}, " \
+            "which is incompatible with the current version, #{Gem.rubygems_version}"
+        end
+      end
+    end
+
+    def lock
+      @definition.lock
+    end
+  end
+end