bundler 1.13.7 → 1.14.0.pre.1

data/lib/bundler/cli/platform.rb

@@ -29,7 +29,7 @@ module Bundler
           output << "Your Gemfile specifies a Ruby version requirement:\n* #{ruby_version}"
 
           begin
-            Bundler.definition.validate_ruby!
+            Bundler.definition.validate_runtime!
             output << "Your current platform satisfies the Ruby version requirement."
           rescue RubyVersionMismatch => e
             output << e.message

data/lib/bundler/cli/show.rb

@@ -13,7 +13,7 @@ module Bundler
 
     def run
       Bundler.ui.silence do
-        Bundler.definition.validate_ruby!
+        Bundler.definition.validate_runtime!
         Bundler.load.lock
       end
 

data/lib/bundler/cli/update.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require "bundler/cli/common"
+
 module Bundler
   class CLI::Update
     attr_reader :options, :gems
@@ -10,7 +12,7 @@ module Bundler
     def run
       Bundler.ui.level = "error" if options[:quiet]
 
-      Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.settings[:plugins]
+      Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.feature_flag.plugins?
 
       sources = Array(options[:source])
       groups  = Array(options[:group]).map(&:to_sym)
@@ -27,7 +29,6 @@ module Bundler
         names = Bundler.locked_gems.specs.map(&:name)
         gems.each do |g|
           next if names.include?(g)
-          require "bundler/cli/common"
           raise GemNotFound, Bundler::CLI::Common.gem_not_found_message(g, names)
         end
 
@@ -36,15 +37,11 @@ module Bundler
           gems.concat(specs.map(&:name))
         end
 
-        Bundler.definition(:gems => gems, :sources => sources, :ruby => options[:ruby])
+        Bundler.definition(:gems => gems, :sources => sources, :ruby => options[:ruby],
+                           :lock_shared_dependencies => options[:conservative])
       end
 
-      patch_level = [:major, :minor, :patch].select {|v| options.keys.include?(v.to_s) }
-      raise ProductionError, "Provide only one of the following options: #{patch_level.join(", ")}" unless patch_level.length <= 1
-      Bundler.definition.gem_version_promoter.tap do |gvp|
-        gvp.level = patch_level.first || :major
-        gvp.strict = options[:strict]
-      end
+      Bundler::CLI::Common.configure_gem_version_promoter(Bundler.definition, options)
 
       Bundler::Fetcher.disable_endpoint = options["full-index"]
 
@@ -54,11 +51,8 @@ module Bundler
 
       Bundler.settings[:jobs] = opts["jobs"] if opts["jobs"]
 
-      # rubygems plugins sometimes hook into the gem install process
-      Gem.load_env_plugins if Gem.respond_to?(:load_env_plugins)
-
-      Bundler.definition.validate_ruby!
-      Installer.install Bundler.root, Bundler.definition, opts
+      Bundler.definition.validate_runtime!
+      installer = Installer.install Bundler.root, Bundler.definition, opts
       Bundler.load.cache if Bundler.app_cache.exist?
 
       if Bundler.settings[:clean] && Bundler.settings[:path]
@@ -67,15 +61,8 @@ module Bundler
       end
 
       Bundler.ui.confirm "Bundle updated!"
-      without_groups_messages
-    end
-
-  private
-
-    def without_groups_messages
-      return unless Bundler.settings.without.any?
-      require "bundler/cli/common"
-      Bundler.ui.confirm Bundler::CLI::Common.without_groups_message
+      Bundler::CLI::Common.output_without_groups_message
+      Bundler::CLI::Common.output_post_install_messages installer.post_install_messages
     end
   end
 end

data/lib/bundler/compact_index_client.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+require "pathname"
+require "set"
+
+module Bundler
+  class CompactIndexClient
+    DEBUG_MUTEX = Mutex.new
+    def self.debug
+      return unless ENV["DEBUG_COMPACT_INDEX"]
+      DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
+    end
+
+    class Error < StandardError; end
+
+    require "bundler/compact_index_client/cache"
+    require "bundler/compact_index_client/updater"
+
+    attr_reader :directory
+
+    # @return [Lambda] A lambda that takes an array of inputs and a block, and
+    #         maps the inputs with the block in parallel.
+    #
+    attr_accessor :in_parallel
+
+    def initialize(directory, fetcher)
+      @directory = Pathname.new(directory)
+      @updater = Updater.new(fetcher)
+      @cache = Cache.new(@directory)
+      @endpoints = Set.new
+      @info_checksums_by_name = {}
+      @parsed_checksums = false
+      @mutex = Mutex.new
+      @in_parallel = lambda do |inputs, &blk|
+        inputs.map(&blk)
+      end
+    end
+
+    def names
+      Bundler::CompactIndexClient.debug { "/names" }
+      update(@cache.names_path, "names")
+      @cache.names
+    end
+
+    def versions
+      Bundler::CompactIndexClient.debug { "/versions" }
+      update(@cache.versions_path, "versions")
+      versions, @info_checksums_by_name = @cache.versions
+      versions
+    end
+
+    def dependencies(names)
+      Bundler::CompactIndexClient.debug { "dependencies(#{names})" }
+      in_parallel.call(names) do |name|
+        update_info(name)
+        @cache.dependencies(name).map {|d| d.unshift(name) }
+      end.flatten(1)
+    end
+
+    def spec(name, version, platform = nil)
+      Bundler::CompactIndexClient.debug { "spec(name = #{name}, version = #{version}, platform = #{platform})" }
+      update_info(name)
+      @cache.specific_dependency(name, version, platform)
+    end
+
+    def update_and_parse_checksums!
+      Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" }
+      return @info_checksums_by_name if @parsed_checksums
+      update(@cache.versions_path, "versions")
+      @info_checksums_by_name = @cache.checksums
+      @parsed_checksums = true
+    end
+
+  private
+
+    def update(local_path, remote_path)
+      Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" }
+      unless synchronize { @endpoints.add?(remote_path) }
+        Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" }
+        return
+      end
+      @updater.update(local_path, url(remote_path))
+    end
+
+    def update_info(name)
+      Bundler::CompactIndexClient.debug { "update_info(#{name})" }
+      path = @cache.info_path(name)
+      checksum = @updater.checksum_for_file(path)
+      unless existing = @info_checksums_by_name[name]
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" }
+        return
+      end
+      if checksum == existing
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" }
+        return
+      end
+      Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" }
+      update(path, "info/#{name}")
+    end
+
+    def url(path)
+      path
+    end
+
+    def synchronize
+      @mutex.synchronize { yield }
+    end
+  end
+end

data/lib/bundler/compact_index_client/cache.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+require "digest/md5"
+
+module Bundler
+  class CompactIndexClient
+    class Cache
+      attr_reader :directory
+
+      def initialize(directory)
+        @directory = Pathname.new(directory).expand_path
+        info_roots.each do |dir|
+          SharedHelpers.filesystem_access(dir) do
+            FileUtils.mkdir_p(dir)
+          end
+        end
+      end
+
+      def names
+        lines(names_path)
+      end
+
+      def names_path
+        directory.join("names")
+      end
+
+      def versions
+        versions_by_name = Hash.new {|hash, key| hash[key] = [] }
+        info_checksums_by_name = {}
+
+        lines(versions_path).each do |line|
+          name, versions_string, info_checksum = line.split(" ", 3)
+          info_checksums_by_name[name] = info_checksum || ""
+          versions_string.split(",").each do |version|
+            if version.start_with?("-")
+              version = version[1..-1].split("-", 2).unshift(name)
+              versions_by_name[name].delete(version)
+            else
+              version = version.split("-", 2).unshift(name)
+              versions_by_name[name] << version
+            end
+          end
+        end
+
+        [versions_by_name, info_checksums_by_name]
+      end
+
+      def versions_path
+        directory.join("versions")
+      end
+
+      def checksums
+        checksums = {}
+
+        lines(versions_path).each do |line|
+          name, _, checksum = line.split(" ", 3)
+          checksums[name] = checksum
+        end
+
+        checksums
+      end
+
+      def dependencies(name)
+        lines(info_path(name)).map do |line|
+          parse_gem(line)
+        end
+      end
+
+      def info_path(name)
+        name = name.to_s
+        if name =~ /[^a-z0-9_-]/
+          name += "-#{Digest::MD5.hexdigest(name).downcase}"
+          info_roots.last.join(name)
+        else
+          info_roots.first.join(name)
+        end
+      end
+
+      def specific_dependency(name, version, platform)
+        pattern = [version, platform].compact.join("-")
+        return nil if pattern.empty?
+
+        gem_lines = info_path(name).read
+        gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0]
+        gem_line ? parse_gem(gem_line) : nil
+      end
+
+    private
+
+      def lines(path)
+        return [] unless path.file?
+        lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n")
+        header = lines.index("---")
+        header ? lines[header + 1..-1] : lines
+      end
+
+      def parse_gem(string)
+        version_and_platform, rest = string.split(" ", 2)
+        version, platform = version_and_platform.split("-", 2)
+        dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+        dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+        requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
+        [version, platform, dependencies, requirements]
+      end
+
+      def parse_dependency(string)
+        dependency = string.split(":")
+        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+        dependency
+      end
+
+      def info_roots
+        [
+          directory.join("info"),
+          directory.join("info-special-characters"),
+        ]
+      end
+    end
+  end
+end

data/lib/bundler/compact_index_client/updater.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+require "fileutils"
+require "stringio"
+require "tmpdir"
+require "zlib"
+
+module Bundler
+  class CompactIndexClient
+    class Updater
+      class MisMatchedChecksumError < Error
+        def initialize(path, server_checksum, local_checksum)
+          @path = path
+          @server_checksum = server_checksum
+          @local_checksum = local_checksum
+        end
+
+        def message
+          "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \
+            "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})."
+        end
+      end
+
+      def initialize(fetcher)
+        @fetcher = fetcher
+      end
+
+      def update(local_path, remote_path, retrying = nil)
+        headers = {}
+
+        Dir.mktmpdir("bundler-compact-index-") do |local_temp_dir|
+          local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename)
+
+          # first try to fetch any new bytes on the existing file
+          if retrying.nil? && local_path.file?
+            FileUtils.cp local_path, local_temp_path
+            headers["If-None-Match"] = etag_for(local_temp_path)
+            headers["Range"] = "bytes=#{local_temp_path.size}-"
+          else
+            # Fastly ignores Range when Accept-Encoding: gzip is set
+            headers["Accept-Encoding"] = "gzip"
+          end
+
+          response = @fetcher.call(remote_path, headers)
+          return nil if response.is_a?(Net::HTTPNotModified)
+
+          content = response.body
+          if response["Content-Encoding"] == "gzip"
+            content = Zlib::GzipReader.new(StringIO.new(content)).read
+          end
+
+          mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w"
+          SharedHelpers.filesystem_access(local_temp_path) do
+            local_temp_path.open(mode) {|f| f << content }
+          end
+
+          response_etag = response["ETag"].gsub(%r{\AW/}, "")
+          if etag_for(local_temp_path) == response_etag
+            SharedHelpers.filesystem_access(local_path) do
+              FileUtils.mv(local_temp_path, local_path)
+            end
+            return nil
+          end
+
+          if retrying
+            raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path))
+          end
+
+          update(local_path, remote_path, :retrying)
+        end
+      end
+
+      def etag_for(path)
+        sum = checksum_for_file(path)
+        sum ? %("#{sum}") : nil
+      end
+
+      def checksum_for_file(path)
+        return nil unless path.file?
+        # This must use IO.read instead of Digest.file().hexdigest
+        # because we need to preserve \n line endings on windows when calculating
+        # the checksum
+        SharedHelpers.filesystem_access(path, :read) do
+          Digest::MD5.hexdigest(IO.read(path))
+        end
+      end
+    end
+  end
+end

data/lib/bundler/current_ruby.rb

@@ -16,6 +16,7 @@ module Bundler
       2.2
       2.3
       2.4
+      2.5
     ).freeze
 
     KNOWN_MAJOR_VERSIONS = KNOWN_MINOR_VERSIONS.map {|v| v.split(".", 2).first }.uniq.freeze
@@ -57,15 +58,15 @@ module Bundler
     end
 
     def mswin64?
-      Bundler::WINDOWS && Gem::Platform.local.os == "mswin64" && Gem::Platform.local.cpu == "x64"
+      Bundler::WINDOWS && Bundler.local_platform.os == "mswin64" && Bundler.local_platform.cpu == "x64"
     end
 
     def mingw?
-      Bundler::WINDOWS && Gem::Platform.local.os == "mingw32" && Gem::Platform.local.cpu != "x64"
+      Bundler::WINDOWS && Bundler.local_platform.os == "mingw32" && Bundler.local_platform.cpu != "x64"
     end
 
     def x64_mingw?
-      Bundler::WINDOWS && Gem::Platform.local.os == "mingw32" && Gem::Platform.local.cpu == "x64"
+      Bundler::WINDOWS && Bundler.local_platform.os == "mingw32" && Bundler.local_platform.cpu == "x64"
     end
 
     (KNOWN_MINOR_VERSIONS + KNOWN_MAJOR_VERSIONS).each do |version|

data/lib/bundler/definition.rb

@@ -62,6 +62,7 @@ module Bundler
       @specs           = nil
       @ruby_version    = ruby_version
 
+      @lockfile               = lockfile
       @lockfile_contents      = String.new
       @locked_bundler_version = nil
       @locked_ruby_version    = nil
@@ -69,7 +70,8 @@ module Bundler
       if lockfile && File.exist?(lockfile)
         @lockfile_contents = Bundler.read_file(lockfile)
         @locked_gems = LockfileParser.new(@lockfile_contents)
-        @platforms = @locked_gems.platforms
+        @locked_platforms = @locked_gems.platforms
+        @platforms = @locked_platforms.dup
         @locked_bundler_version = @locked_gems.bundler_version
         @locked_ruby_version = @locked_gems.ruby_version
 
@@ -90,25 +92,24 @@ module Bundler
         @locked_deps    = []
         @locked_specs   = SpecSet.new([])
         @locked_sources = []
+        @locked_platforms = []
       end
 
       @unlock[:gems] ||= []
       @unlock[:sources] ||= []
-      @unlock[:ruby] ||= if @ruby_version && @locked_ruby_version
-        unless locked_ruby_version_object = RubyVersion.from_string(@locked_ruby_version)
-          raise LockfileError, "Failed to create a `RubyVersion` object from " \
-            "`#{@locked_ruby_version}` found in #{lockfile} -- try running `bundle update --ruby`."
-        end
+      @unlock[:ruby] ||= if @ruby_version && locked_ruby_version_object
         @ruby_version.diff(locked_ruby_version_object)
       end
       @unlocking ||= @unlock[:ruby] ||= (!@locked_ruby_version ^ !@ruby_version)
 
-      current_platform = Bundler.rubygems.platforms.map {|p| generic(p) }.compact.last
-      add_platform(current_platform)
+      add_current_platform unless Bundler.settings[:frozen]
 
       @path_changes = converge_paths
-      eager_unlock = expand_dependencies(@unlock[:gems])
-      @unlock[:gems] = @locked_specs.for(eager_unlock).map(&:name)
+
+      unless @unlock[:lock_shared_dependencies]
+        eager_unlock = expand_dependencies(@unlock[:gems])
+        @unlock[:gems] = @locked_specs.for(eager_unlock).map(&:name)
+      end
 
       @gem_version_promoter = create_gem_version_promoter
 
@@ -175,7 +176,7 @@ module Bundler
         rescue GemNotFound => e # Handle yanked gem
           gem_name, gem_version = extract_gem_info(e)
           locked_gem = @locked_specs[gem_name].last
-          raise if locked_gem.nil? || locked_gem.version.to_s != gem_version
+          raise if locked_gem.nil? || locked_gem.version.to_s != gem_version || !@remote
           raise GemNotFound, "Your bundle is locked to #{locked_gem}, but that version could not " \
                              "be found in any of the sources listed in your Gemfile. If you haven't changed sources, " \
                              "that means the author of #{locked_gem} has removed it. You'll need to update your bundle " \
@@ -247,7 +248,7 @@ module Bundler
         else
           # Run a resolve against the locally available gems
           Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}")
-          last_resolve.merge Resolver.resolve(expanded_dependencies, index, source_requirements, last_resolve, ruby_version, gem_version_promoter, additional_base_requirements_for_resolve)
+          last_resolve.merge Resolver.resolve(expanded_dependencies, index, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve)
         end
       end
     end
@@ -262,6 +263,8 @@ module Bundler
           dependency_names -= pinned_spec_names(source.specs)
           dependency_names.concat(source.unmet_deps).uniq!
         end
+        idx << Gem::Specification.new("ruby\0", RubyVersion.system.to_gem_version_with_patchlevel)
+        idx << Gem::Specification.new("rubygems\0", Gem::VERSION)
       end
     end
 
@@ -338,6 +341,18 @@ module Bundler
       end
     end
 
+    def locked_ruby_version_object
+      return unless @locked_ruby_version
+      @locked_ruby_version_object ||= begin
+        unless version = RubyVersion.from_string(@locked_ruby_version)
+          raise LockfileError, "The Ruby version #{@locked_ruby_version} from " \
+            "#{@lockfile} could not be parsed. " \
+            "Try running bundle update --ruby to resolve this."
+        end
+        version
+      end
+    end
+
     def to_lock
       out = String.new
 
@@ -401,6 +416,11 @@ module Bundler
       deleted = []
       changed = []
 
+      new_platforms = @platforms - @locked_platforms
+      deleted_platforms = @locked_platforms - @platforms
+      added.concat new_platforms.map {|p| "* platform: #{p}" }
+      deleted.concat deleted_platforms.map {|p| "* platform: #{p}" }
+
       gemfile_sources = sources.lock_sources
 
       new_sources = gemfile_sources - @locked_sources
@@ -449,6 +469,11 @@ module Bundler
       raise ProductionError, msg if added.any? || deleted.any? || changed.any?
     end
 
+    def validate_runtime!
+      validate_ruby!
+      validate_platforms!
+    end
+
     def validate_ruby!
       return unless ruby_version
 
@@ -474,6 +499,18 @@ module Bundler
       end
     end
 
+    def validate_platforms!
+      return if @platforms.any? do |bundle_platform|
+        Bundler.rubygems.platforms.any? do |local_platform|
+          MatchPlatform.platforms_match?(bundle_platform, local_platform)
+        end
+      end
+
+      raise ProductionError, "Your bundle only supports platforms #{@platforms.map(&:to_s)} " \
+        "but your local platforms are #{Bundler.rubygems.platforms.map(&:to_s)}, and " \
+        "there's no compatible match between those two lists."
+    end
+
     def add_platform(platform)
       @new_platform ||= !@platforms.include?(platform)
       @platforms |= [platform]
@@ -484,6 +521,20 @@ module Bundler
       raise InvalidOption, "Unable to remove the platform `#{platform}` since the only platforms are #{@platforms.join ", "}"
     end
 
+    def add_current_platform
+      current_platform = Bundler.local_platform
+      add_platform(current_platform) if Bundler.settings[:specific_platform]
+      add_platform(generic(current_platform))
+    end
+
+    def find_resolved_spec(current_spec)
+      specs.find_by_name_and_platform(current_spec.name, current_spec.platform)
+    end
+
+    def find_indexed_specs(current_spec)
+      index[current_spec.name].select {|spec| spec.match_platform(current_spec.platform) }.sort_by(&:version)
+    end
+
     attr_reader :sources
     private :sources
 
@@ -726,16 +777,54 @@ module Bundler
       @locked_specs.any? {|s| s.satisfies?(dep) && (!dep.source || s.source.include?(dep.source)) }
     end
 
+    # This list of dependencies is only used in #resolve, so it's OK to add
+    # the metadata dependencies here
     def expanded_dependencies
-      @expanded_dependencies ||= expand_dependencies(dependencies, @remote)
+      @expanded_dependencies ||= begin
+        ruby_versions = concat_ruby_version_requirements(@ruby_version)
+        if ruby_versions.empty? || !@ruby_version.exact?
+          concat_ruby_version_requirements(RubyVersion.system)
+          concat_ruby_version_requirements(locked_ruby_version_object) unless @unlock[:ruby]
+        end
+
+        metadata_dependencies = [
+          Dependency.new("ruby\0", ruby_versions),
+          Dependency.new("rubygems\0", Gem::VERSION),
+        ]
+        expand_dependencies(dependencies + metadata_dependencies, @remote)
+      end
+    end
+
+    def concat_ruby_version_requirements(ruby_version, ruby_versions = [])
+      return ruby_versions unless ruby_version
+      if ruby_version.patchlevel
+        ruby_versions << ruby_version.to_gem_version_with_patchlevel
+      else
+        ruby_versions.concat(ruby_version.versions.map do |version|
+          requirement = Gem::Requirement.new(version)
+          if requirement.exact?
+            "~> #{version}.0"
+          else
+            requirement
+          end
+        end)
+      end
     end
 
     def expand_dependencies(dependencies, remote = false)
       deps = []
       dependencies.each do |dep|
         dep = Dependency.new(dep, ">= 0") unless dep.respond_to?(:name)
-        next unless remote || dep.current_platform?
-        dep.gem_platforms(@platforms).each do |p|
+        next if !remote && !dep.current_platform?
+        platforms = dep.gem_platforms(@platforms)
+        if platforms.empty?
+          Bundler.ui.warn \
+            "The dependency #{dep} will be unused by any of the platforms Bundler is installing for. " \
+            "Bundler is installing for #{@platforms.join ", "} but the dependency " \
+            "is only for #{dep.platforms.map {|p| Dependency::PLATFORM_MAP[p] }.join ", "}. " \
+            "To add those platforms to the bundle, run `bundle lock --add-platform #{dep.platforms.join ", "}`."
+        end
+        platforms.each do |p|
           deps << DepProxy.new(dep, p) if remote || p == generic_local_platform
         end
       end
@@ -812,9 +901,10 @@ module Bundler
     end
 
     def additional_base_requirements_for_resolve
-      return [] unless @locked_gems && Bundler.settings[:only_update_to_newer_versions]
+      return [] unless @locked_gems && Bundler.feature_flag.only_update_to_newer_versions?
       @locked_gems.specs.reduce({}) do |requirements, locked_spec|
-        requirements[locked_spec.name] = Gem::Dependency.new(locked_spec.name, ">= #{locked_spec.version}")
+        dep = Gem::Dependency.new(locked_spec.name, ">= #{locked_spec.version}")
+        requirements[locked_spec.name] = DepProxy.new(dep, locked_spec.platform)
         requirements
       end.values
     end