bundler 1.13.6 → 1.17.3

data/lib/bundler/installer.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "erb"
 require "rubygems/dependency_installer"
 require "bundler/worker"
@@ -20,8 +21,9 @@ module Bundler
     # For more information see the #run method on this class.
     def self.install(root, definition, options = {})
       installer = new(root, definition)
-      Plugin.hook("before-install-all", definition.dependencies)
+      Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL_ALL, definition.dependencies)
       installer.run(options)
+      Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL_ALL, definition.dependencies)
       installer
     end
 
@@ -33,25 +35,26 @@ module Bundler
 
     # Runs the install procedures for a specific Gemfile.
     #
-    # Firstly, this method will check to see if Bundler.bundle_path exists
-    # and if not then will create it. This is usually the location of gems
-    # on the system, be it RVM or at a system path.
+    # Firstly, this method will check to see if `Bundler.bundle_path` exists
+    # and if not then Bundler will create the directory. This is usually the same
+    # location as RubyGems which typically is the `~/.gem` directory
+    # unless other specified.
     #
-    # Secondly, it checks if Bundler has been configured to be "frozen"
+    # Secondly, it checks if Bundler has been configured to be "frozen".
     # Frozen ensures that the Gemfile and the Gemfile.lock file are matching.
     # This stops a situation where a developer may update the Gemfile but may not run
     # `bundle install`, which leads to the Gemfile.lock file not being correctly updated.
     # If this file is not correctly updated then any other developer running
     # `bundle install` will potentially not install the correct gems.
     #
-    # Thirdly, Bundler checks if there are any dependencies specified in the Gemfile using
-    # Bundler::Environment#dependencies. If there are no dependencies specified then
-    # Bundler returns a warning message stating so and this method returns.
+    # Thirdly, Bundler checks if there are any dependencies specified in the Gemfile.
+    # If there are no dependencies specified then Bundler returns a warning message stating
+    # so and this method returns.
     #
-    # Fourthly, Bundler checks if the default lockfile (Gemfile.lock) exists, and if so
-    # then proceeds to set up a defintion based on the default gemfile (Gemfile) and the
-    # default lock file (Gemfile.lock). However, this is not the case if the platform is different
-    # to that which is specified in Gemfile.lock, or if there are any missing specs for the gems.
+    # Fourthly, Bundler checks if the Gemfile.lock exists, and if so
+    # then proceeds to set up a definition based on the Gemfile and the Gemfile.lock.
+    # During this step Bundler will also download information about any new gems
+    # that are not in the Gemfile.lock and resolve any dependencies if needed.
     #
     # Fifthly, Bundler resolves the dependencies either through a cache of gems or by remote.
     # This then leads into the gems being installed, along with stubs for their executables,
@@ -61,26 +64,36 @@ module Bundler
     # Sixthly, a new Gemfile.lock is created from the installed gems to ensure that the next time
     # that a user runs `bundle install` they will receive any updates from this process.
     #
-    # Finally: TODO add documentation for how the standalone process works.
+    # Finally, if the user has specified the standalone flag, Bundler will generate the needed
+    # require paths and save them in a `setup.rb` file. See `bundle standalone --help` for more
+    # information.
     def run(options)
       create_bundle_path
 
-      if Bundler.settings[:frozen]
-        @definition.ensure_equivalent_gemfile_and_lockfile(options[:deployment])
-      end
+      ProcessLock.lock do
+        if Bundler.frozen_bundle?
+          @definition.ensure_equivalent_gemfile_and_lockfile(options[:deployment])
+        end
 
-      if @definition.dependencies.empty?
-        Bundler.ui.warn "The Gemfile specifies no dependencies"
-        lock
-        return
-      end
+        if @definition.dependencies.empty?
+          Bundler.ui.warn "The Gemfile specifies no dependencies"
+          lock
+          return
+        end
 
-      resolve_if_need(options)
-      ensure_specs_are_compatible!
-      install(options)
+        if resolve_if_needed(options)
+          ensure_specs_are_compatible!
+          warn_on_incompatible_bundler_deps
+          load_plugins
+          options.delete(:jobs)
+        else
+          options[:jobs] = 1 # to avoid the overhead of Bundler::Worker
+        end
+        install(options)
 
-      lock unless Bundler.settings[:frozen]
-      Standalone.new(options[:standalone], @definition).generate if options[:standalone]
+        lock unless Bundler.frozen_bundle?
+        Standalone.new(options[:standalone], @definition).generate if options[:standalone]
+      end
     end
 
     def generate_bundler_executable_stubs(spec, options = {})
@@ -101,15 +114,21 @@ module Bundler
       end
 
       # double-assignment to avoid warnings about variables that will be used by ERB
-      bin_path = bin_path = Bundler.bin_path
-      template = template = File.read(File.expand_path("../templates/Executable", __FILE__))
-      relative_gemfile_path = relative_gemfile_path = Bundler.default_gemfile.relative_path_from(bin_path)
-      ruby_command = ruby_command = Thor::Util.ruby_command
+      bin_path = Bundler.bin_path
+      bin_path = bin_path
+      relative_gemfile_path = Bundler.default_gemfile.relative_path_from(bin_path)
+      relative_gemfile_path = relative_gemfile_path
+      ruby_command = Thor::Util.ruby_command
+      ruby_command = ruby_command
+      template_path = File.expand_path("../templates/Executable", __FILE__)
+      if spec.name == "bundler"
+        template_path += ".bundler"
+        spec.executables = %(bundle)
+      end
+      template = File.read(template_path)
 
       exists = []
       spec.executables.each do |executable|
-        next if executable == "bundle"
-
         binstub_path = "#{bin_path}/#{executable}"
         if File.exist?(binstub_path) && !options[:force]
           exists << executable
@@ -117,7 +136,11 @@ module Bundler
         end
 
         File.open(binstub_path, "w", 0o777 & ~File.umask) do |f|
-          f.puts ERB.new(template, nil, "-").result(binding)
+          if RUBY_VERSION >= "2.6"
+            f.puts ERB.new(template, :trim_mode => "-").result(binding)
+          else
+            f.puts ERB.new(template, nil, "-").result(binding)
+          end
         end
       end
 
@@ -139,15 +162,25 @@ module Bundler
     def generate_standalone_bundler_executable_stubs(spec)
       # double-assignment to avoid warnings about variables that will be used by ERB
       bin_path = Bundler.bin_path
-      standalone_path = standalone_path = Bundler.root.join(Bundler.settings[:path]).relative_path_from(bin_path)
+      unless path = Bundler.settings[:path]
+        raise "Can't standalone without an explicit path set"
+      end
+      standalone_path = Bundler.root.join(path).relative_path_from(bin_path)
+      standalone_path = standalone_path
       template = File.read(File.expand_path("../templates/Executable.standalone", __FILE__))
-      ruby_command = ruby_command = Thor::Util.ruby_command
+      ruby_command = Thor::Util.ruby_command
+      ruby_command = ruby_command
 
       spec.executables.each do |executable|
         next if executable == "bundle"
-        executable_path = executable_path = Pathname(spec.full_gem_path).join(spec.bindir, executable).relative_path_from(bin_path)
+        executable_path = Pathname(spec.full_gem_path).join(spec.bindir, executable).relative_path_from(bin_path)
+        executable_path = executable_path
         File.open "#{bin_path}/#{executable}", "w", 0o755 do |f|
-          f.puts ERB.new(template, nil, "-").result(binding)
+          if RUBY_VERSION >= "2.6"
+            f.puts ERB.new(template, :trim_mode => "-").result(binding)
+          else
+            f.puts ERB.new(template, nil, "-").result(binding)
+          end
         end
       end
     end
@@ -160,11 +193,53 @@ module Bundler
     # installation is SO MUCH FASTER. so we let people opt in.
     def install(options)
       force = options["force"]
-      jobs = 1
-      jobs = [Bundler.settings[:jobs].to_i - 1, 1].max if can_install_in_parallel?
+      jobs = installation_parallelization(options)
       install_in_parallel jobs, options[:standalone], force
     end
 
+    def installation_parallelization(options)
+      if jobs = options.delete(:jobs)
+        return jobs
+      end
+
+      return 1 unless can_install_in_parallel?
+
+      auto_config_jobs = Bundler.feature_flag.auto_config_jobs?
+      if jobs = Bundler.settings[:jobs]
+        if auto_config_jobs
+          jobs
+        else
+          [jobs.pred, 1].max
+        end
+      elsif auto_config_jobs
+        processor_count
+      else
+        1
+      end
+    end
+
+    def processor_count
+      require "etc"
+      Etc.nprocessors
+    rescue
+      1
+    end
+
+    def load_plugins
+      Bundler.rubygems.load_plugins
+
+      requested_path_gems = @definition.requested_specs.select {|s| s.source.is_a?(Source::Path) }
+      path_plugin_files = requested_path_gems.map do |spec|
+        begin
+          Bundler.rubygems.spec_matches_for_glob(spec, "rubygems_plugin#{Bundler.rubygems.suffix_pattern}")
+        rescue TypeError
+          error_message = "#{spec.name} #{spec.version} has an invalid gemspec"
+          raise Gem::InvalidSpecificationException, error_message
+        end
+      end.flatten
+      Bundler.rubygems.load_plugin_files(path_plugin_files)
+    end
+
     def ensure_specs_are_compatible!
       system_ruby = Bundler::RubyVersion.system
       rubygems_version = Gem::Version.create(Gem::VERSION)
@@ -183,12 +258,28 @@ module Bundler
       end
     end
 
+    def warn_on_incompatible_bundler_deps
+      bundler_version = Gem::Version.create(Bundler::VERSION)
+      @definition.specs.each do |spec|
+        spec.dependencies.each do |dep|
+          next if dep.type == :development
+          next unless dep.name == "bundler".freeze
+          next if dep.requirement.satisfied_by?(bundler_version)
+
+          Bundler.ui.warn "#{spec.name} (#{spec.version}) has dependency" \
+            " #{SharedHelpers.pretty_dependency(dep)}" \
+            ", which is unsatisfied by the current bundler version #{VERSION}" \
+            ", so the dependency is being ignored"
+        end
+      end
+    end
+
     def can_install_in_parallel?
       if Bundler.rubygems.provides?(">= 2.1.0")
         true
       else
-        Bundler.ui.warn "Rubygems #{Gem::VERSION} is not threadsafe, so your "\
-          "gems will be installed one at a time. Upgrade to Rubygems 2.1.0 " \
+        Bundler.ui.warn "RubyGems #{Gem::VERSION} is not threadsafe, so your "\
+          "gems will be installed one at a time. Upgrade to RubyGems 2.1.0 " \
           "or higher to enable parallel gem installation."
         false
       end
@@ -206,23 +297,18 @@ module Bundler
         Bundler.mkdir_p(p)
       end unless Bundler.bundle_path.exist?
     rescue Errno::EEXIST
-      raise PathError, "Could not install to path `#{Bundler.settings[:path]}` " \
-        "because of an invalid symlink. Remove the symlink so the directory can be created."
+      raise PathError, "Could not install to path `#{Bundler.bundle_path}` " \
+        "because a file already exists at that path. Either remove or rename the file so the directory can be created."
     end
 
-    def resolve_if_need(options)
-      if Bundler.default_lockfile.exist? && !options["update"]
-        local = Bundler.ui.silence do
-          begin
-            tmpdef = Definition.build(Bundler.default_gemfile, Bundler.default_lockfile, nil)
-            true unless tmpdef.new_platform? || tmpdef.missing_dependencies.any?
-          rescue BundlerError
-          end
-        end
+    # returns whether or not a re-resolve was needed
+    def resolve_if_needed(options)
+      if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file?
+        return false if @definition.nothing_changed? && !@definition.missing_specs?
       end
 
-      return if local
       options["local"] ? @definition.resolve_with_cache! : @definition.resolve_remotely!
+      true
     end
 
     def lock(opts = {})

data/lib/bundler/installer/gem_installer.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Bundler
   class GemInstaller
     attr_reader :spec, :standalone, :worker, :force, :installer
@@ -16,11 +17,11 @@ module Bundler
       Bundler.ui.debug "#{worker}:  #{spec.name} (#{spec.version}) from #{spec.loaded_from}"
       generate_executable_stubs
       return true, post_install_message
-    rescue Bundler::InstallHookError, Bundler::SecurityError
+    rescue Bundler::InstallHookError, Bundler::SecurityError, APIResponseMismatchError
       raise
     rescue Errno::ENOSPC
       return false, out_of_space_message
-    rescue => e
+    rescue StandardError => e
       return false, specific_failure_message(e)
     end
 
@@ -43,7 +44,14 @@ module Bundler
     end
 
     def gem_install_message
-      "Make sure that `gem install #{spec.name} -v '#{spec.version}'` succeeds before bundling."
+      source = spec.source
+      return unless source.respond_to?(:remotes)
+
+      if source.remotes.size == 1
+        "Make sure that `gem install #{spec.name} -v '#{spec.version}' --source '#{source.remotes.first}'` succeeds before bundling."
+      else
+        "Make sure that `gem install #{spec.name} -v '#{spec.version}'` succeeds before bundling."
+      end
     end
 
     def spec_settings
@@ -52,12 +60,12 @@ module Bundler
     end
 
     def install
-      spec.source.install(spec, :force => force, :ensure_builtin_gems_cached => standalone)
+      spec.source.install(spec, :force => force, :ensure_builtin_gems_cached => standalone, :build_args => Array(spec_settings))
     end
 
     def install_with_settings
       # Build arguments are global, so this is mutexed
-      Bundler.rubygems.with_build_args([spec_settings]) { install }
+      Bundler.rubygems.install_with_build_args([spec_settings]) { install }
     end
 
     def out_of_space_message
@@ -65,6 +73,8 @@ module Bundler
     end
 
     def generate_executable_stubs
+      return if Bundler.feature_flag.forget_cli_options?
+      return if Bundler.settings[:inline]
       if Bundler.settings[:bin] && standalone
         installer.generate_standalone_bundler_executable_stubs(spec)
       elsif Bundler.settings[:bin]

data/lib/bundler/installer/parallel_installer.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "bundler/worker"
 require "bundler/installer/gem_installer"
 
@@ -47,37 +48,37 @@ module Bundler
       # sure needed dependencies have been installed.
       def dependencies_installed?(all_specs)
         installed_specs = all_specs.select(&:installed?).map(&:name)
-        dependencies(all_specs.map(&:name)).all? {|d| installed_specs.include? d.name }
+        dependencies.all? {|d| installed_specs.include? d.name }
       end
 
       # Represents only the non-development dependencies, the ones that are
       # itself and are in the total list.
-      def dependencies(all_spec_names)
+      def dependencies
         @dependencies ||= begin
-          deps = all_dependencies.reject {|dep| ignorable_dependency? dep }
-          missing = deps.reject {|dep| all_spec_names.include? dep.name }
-          unless missing.empty?
-            raise Bundler::LockfileError, "Your Gemfile.lock is corrupt. The following #{missing.size > 1 ? "gems are" : "gem is"} missing " \
-                                "from the DEPENDENCIES section: '#{missing.map(&:name).join('\' \'')}'"
-          end
-          deps
+          all_dependencies.reject {|dep| ignorable_dependency? dep }
         end
       end
 
+      def missing_lockfile_dependencies(all_spec_names)
+        deps = all_dependencies.reject {|dep| ignorable_dependency? dep }
+        deps.reject {|dep| all_spec_names.include? dep.name }
+      end
+
       # Represents all dependencies
       def all_dependencies
         @spec.dependencies
       end
+
+      def to_s
+        "#<#{self.class} #{@spec.full_name} (#{state})>"
+      end
     end
 
     def self.call(*args)
       new(*args).call
     end
 
-    # Returns max number of threads machine can handle with a min of 1
-    def self.max_threads
-      [Bundler.settings[:jobs].to_i - 1, 1].max
-    end
+    attr_reader :size
 
     def initialize(installer, all_specs, size, standalone, force)
       @installer = installer
@@ -85,44 +86,113 @@ module Bundler
       @standalone = standalone
       @force = force
       @specs = all_specs.map {|s| SpecInstallation.new(s) }
+      @spec_set = all_specs
+      @rake = @specs.find {|s| s.name == "rake" }
     end
 
     def call
-      enqueue_specs
-      process_specs until @specs.all?(&:installed?) || @specs.any?(&:failed?)
+      # Since `autoload` has the potential for threading issues on 1.8.7
+      # TODO:  remove in bundler 2.0
+      require "bundler/gem_remote_fetcher" if RUBY_VERSION < "1.9"
+
+      check_for_corrupt_lockfile
+
+      if @size > 1
+        install_with_worker
+      else
+        install_serially
+      end
+
       handle_error if @specs.any?(&:failed?)
       @specs
     ensure
       worker_pool && worker_pool.stop
     end
 
+    def check_for_corrupt_lockfile
+      missing_dependencies = @specs.map do |s|
+        [
+          s,
+          s.missing_lockfile_dependencies(@specs.map(&:name)),
+        ]
+      end.reject { |a| a.last.empty? }
+      return if missing_dependencies.empty?
+
+      warning = []
+      warning << "Your lockfile was created by an old Bundler that left some things out."
+      if @size != 1
+        warning << "Because of the missing DEPENDENCIES, we can only install gems one at a time, instead of installing #{@size} at a time."
+        @size = 1
+      end
+      warning << "You can fix this by adding the missing gems to your Gemfile, running bundle install, and then removing the gems from your Gemfile."
+      warning << "The missing gems are:"
+
+      missing_dependencies.each do |spec, missing|
+        warning << "* #{missing.map(&:name).join(", ")} depended upon by #{spec.name}"
+      end
+
+      Bundler.ui.warn(warning.join("\n"))
+    end
+
+  private
+
+    def install_with_worker
+      enqueue_specs
+      process_specs until finished_installing?
+    end
+
+    def install_serially
+      until finished_installing?
+        raise "failed to find a spec to enqueue while installing serially" unless spec_install = @specs.find(&:ready_to_enqueue?)
+        spec_install.state = :enqueued
+        do_install(spec_install, 0)
+      end
+    end
+
     def worker_pool
       @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda { |spec_install, worker_num|
-        gem_installer = Bundler::GemInstaller.new(
-          spec_install.spec, @installer, @standalone, worker_num, @force
-        )
-        success, message = gem_installer.install_from_spec
-        if success && !message.nil?
-          spec_install.post_install_message = message
-        elsif !success
-          spec_install.state = :failed
-          spec_install.error = message
-        end
-        spec_install
+        do_install(spec_install, worker_num)
       }
     end
 
+    def do_install(spec_install, worker_num)
+      Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL, spec_install)
+      gem_installer = Bundler::GemInstaller.new(
+        spec_install.spec, @installer, @standalone, worker_num, @force
+      )
+      success, message = begin
+        gem_installer.install_from_spec
+      rescue RuntimeError => e
+        raise e, "#{e}\n\n#{require_tree_for_spec(spec_install.spec)}"
+      end
+      if success
+        spec_install.state = :installed
+        spec_install.post_install_message = message unless message.nil?
+      else
+        spec_install.state = :failed
+        spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}"
+      end
+      Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install)
+      spec_install
+    end
+
     # Dequeue a spec and save its post-install message and then enqueue the
     # remaining specs.
     # Some specs might've had to wait til this spec was installed to be
     # processed so the call to `enqueue_specs` is important after every
     # dequeue.
     def process_specs
-      spec = worker_pool.deq
-      spec.state = :installed unless spec.failed?
+      worker_pool.deq
       enqueue_specs
     end
 
+    def finished_installing?
+      @specs.all? do |spec|
+        return true if spec.failed?
+        spec.installed?
+      end
+    end
+
     def handle_error
       errors = @specs.select(&:failed?).map(&:error)
       if exception = errors.find {|e| e.is_a?(Bundler::BundlerError) }
@@ -131,6 +201,19 @@ module Bundler
       raise Bundler::InstallError, errors.map(&:to_s).join("\n\n")
     end
 
+    def require_tree_for_spec(spec)
+      tree = @spec_set.what_required(spec)
+      t = String.new("In #{File.basename(SharedHelpers.default_gemfile)}:\n")
+      tree.each_with_index do |s, depth|
+        t << "  " * depth.succ << s.name
+        unless tree.last == s
+          t << %( was resolved to #{s.version}, which depends on)
+        end
+        t << %(\n)
+      end
+      t
+    end
+
     # Keys in the remains hash represent uninstalled gems specs.
     # We enqueue all gem specs that do not have any dependencies.
     # Later we call this lambda again to install specs that depended on
@@ -138,6 +221,8 @@ module Bundler
     # are installed.
     def enqueue_specs
       @specs.select(&:ready_to_enqueue?).each do |spec|
+        next if @rake && !@rake.installed? && spec.name != @rake.name
+
         if spec.dependencies_installed? @specs
           spec.state = :enqueued
           worker_pool.enq spec