bundler-multilock 2.0.0.beta4 → 2.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45718b8a0410e25b206212808dd21eb5530561297393c3f6fca54a14acd26e40
-  data.tar.gz: 35245c2fc0f936995c0cbc5be482bf20dc6767de70cc23b41fe1b89b63e179cd
+  metadata.gz: 730bc7b4e1a107a6b3fd9527014b630b97918ed317e703b8cf045c67d6f6fae9
+  data.tar.gz: 11943fc5875e3a07ba89f66b38e6b4ab6d100fe0d47168e24d5d3ae023609ebb
 SHA512:
-  metadata.gz: 2e069c0c770748ebdfe957fc1c795553ecaa7afdd7655a180e5c6ced84f0e31840bd7f9233dcd6d07b70ac765fbde244624b1976a15f2d5dc31adb1e1accadaf
-  data.tar.gz: c70adec856e83bb4520d3705e714bd2d1e99672d8c866faf1718928d65c1b3062b004c242e353037f7d7a0b35ea7bfb0675b2af20afd64416c1f8a14d03c7421
+  metadata.gz: 998f0024a84771dc5d61ada77c069bd267c9d0da5c1d09acd11c098a02bdcd6c537a32e5e8bfd3005ebb9050d1fea9afed8a6f4cc32877e5af8224089431cd00
+  data.tar.gz: dcf0c9c3e0089f48f796857eec57f555381ff215327ba904a11ef294c1b8bd45611613be3548b8def4c0dc7a4abd6f2fa65f7082251bfe83d216c092891a822e

data/lib/bundler/multilock/ext/dsl.rb

@@ -99,10 +99,15 @@ module Bundler
           @allowed_duplicate_plugins = Set.new
         end
 
-        def lockfile(*, **, &)
+        # Significant changes:
+        #  * a lockfile with a block or options is managed by Multilock
+        #  * a bare lockfile name is just Bundler's own `lockfile`, for
+        #    Gemfiles that only want to name their lockfile
+        def lockfile(*args, **kwargs, &block)
           return true if Multilock.loaded?
+          return super(args.first) if args.length == 1 && args.first.is_a?(String) && kwargs.empty? && !block
 
-          Multilock.add_lockfile(*, builder: self, **, &)
+          Multilock.add_lockfile(*args, builder: self, **kwargs, &block)
         end
       end
     end

data/lib/bundler/multilock/ext/runtime.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Bundler
+  module Multilock
+    module Ext
+      # Makes `bundle cache` (and `bundle install` when vendor/cache exists) package
+      # the gems from every lockfile, and only prune gems that none of them use.
+      # Likewise, `bundle clean` only removes gems that none of the lockfiles use, and the
+      # `prune` setting covers the git gems from all of them.
+      module Runtime
+        ::Bundler::Runtime.prepend(self)
+
+        def cache(custom_path = nil, *)
+          cache_alternate_lockfiles(custom_path)
+          super
+        end
+
+        def prune_cache(cache_path)
+          return super if multilock_alternate_definitions.empty?
+
+          ensure_multilock_cache_path(cache_path)
+
+          resolve = [@definition, *multilock_alternate_definitions].flat_map { |definition| definition.resolve.to_a }
+          prune_gem_cache(resolve, cache_path)
+          prune_git_and_path_cache(resolve, cache_path)
+        end
+
+        # Bundler's `clean` only keeps what the one definition needs, so keep what all of them need.
+        # It also keys the specs it keeps by name, which would drop all but one version of a gem
+        # that the lockfiles disagree on, so that's replaced for the duration. (Keyed by where they
+        # are installed, since different lockfiles can lock different revisions of the same git gem.)
+        def clean(*)
+          alternate_definitions = multilock_alternate_definitions
+          return super if alternate_definitions.empty?
+
+          combined = [specs, *alternate_definitions.map(&:specs)].flat_map(&:to_a)
+          rubygems = Bundler.rubygems
+          original_add_default_gems_to = rubygems.method(:add_default_gems_to)
+          rubygems.define_singleton_method(:add_default_gems_to) do |_specs|
+            (original_add_default_gems_to.call([]).values + combined).to_h { |spec| [spec.full_gem_path, spec] }
+          end
+
+          begin
+            super
+          ensure
+            rubygems.singleton_class.send(:remove_method, :add_default_gems_to)
+          end
+        end
+
+        private
+
+        # Same as Bundler's, except that it also covers the git gems that only other lockfiles use
+        def prune_git_metadata
+          return super if multilock_alternate_definitions.empty?
+
+          owned = "#{Bundler.install_path}#{File::SEPARATOR}"
+          git_sources = [@definition, *multilock_alternate_definitions].flat_map do |definition|
+            definition.sources.git_sources
+          end
+          git_dirs = git_sources.reject(&:local?).filter_map do |source|
+            install_path = source.install_path.to_s
+            next unless install_path.start_with?(owned)
+
+            git_dir = File.join(install_path, ".git")
+            git_dir if File.exist?(git_dir)
+          end.uniq
+          return if git_dirs.empty?
+
+          Bundler.ui.info "Removing git metadata from checked out git gems"
+          git_dirs.each do |git_dir|
+            ::Bundler::SharedHelpers.filesystem_access(git_dir) do |p|
+              FileUtils.rm_rf(p)
+            end
+          end
+        end
+
+        def ensure_multilock_cache_path(cache_path)
+          return if File.exist?(cache_path)
+
+          ::Bundler::SharedHelpers.filesystem_access(cache_path) do |p|
+            FileUtils.mkdir_p(p)
+          end
+        end
+
+        # @return [Array<Definition>] the definitions of the lockfiles other than the one being cached
+        def multilock_alternate_definitions
+          @multilock_alternate_definitions ||= Multilock.alternate_definitions
+        end
+
+        # Caches the gems that only the other lockfiles use. The ones in common are cached by the
+        # usual process, which also cleans up git sources and prunes afterwards.
+        def cache_alternate_lockfiles(custom_path)
+          return if multilock_alternate_definitions.empty?
+
+          cache_path = Bundler.app_cache(custom_path)
+          ensure_multilock_cache_path(cache_path)
+
+          cached = Set.new
+          multilock_alternate_definitions.each do |definition|
+            specs_to_cache = if Bundler.settings[:cache_all_platforms]
+                               definition.resolve.materialized_for_all_platforms
+                             else
+                               definition.specs
+                             end
+
+            specs_to_cache.each do |spec|
+              next if spec.name == "bundler"
+
+              source = spec.source
+              next if source.is_a?(::Bundler::Source::Gemspec)
+
+              # git sources are equal regardless of which revision is locked, but each one
+              # is cached separately
+              cache_key = source.respond_to?(:app_cache_dirname) ? source.app_cache_dirname : source
+              next unless cached.add?([spec.full_name, cache_key])
+
+              if source.respond_to?(:migrate_cache)
+                source.migrate_cache(custom_path, local: false)
+              elsif source.respond_to?(:cache)
+                source.cache(spec, custom_path)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bundler/multilock/version.rb

@@ -2,6 +2,6 @@
 
 module Bundler
   module Multilock
-    VERSION = "2.0.0.beta4"
+    VERSION = "2.0.0.beta5"
   end
 end

data/lib/bundler/multilock.rb

@@ -4,6 +4,7 @@ require_relative "multilock/ext/bundler"
 require_relative "multilock/ext/definition"
 require_relative "multilock/ext/dsl"
 require_relative "multilock/ext/plugin/dsl"
+require_relative "multilock/ext/runtime"
 require_relative "multilock/ext/shared_helpers"
 require_relative "multilock/ext/source"
 require_relative "multilock/ext/source_list"
@@ -40,11 +41,15 @@ module Bundler
                        parent: nil,
                        enforce_pinned_additional_dependencies: false,
                        &block)
+        # remembered for gems that need to be kept for the primary lockfile, even if it was never
+        # declared and an alternate lockfile is active (which may use its own Gemfile)
+        @primary_gemfile ||= Bundler.default_gemfile
+        @primary_lockfile ||= Bundler.default_lockfile(force_original: true)
+
         active = true if active.nil? && lockfile_definitions.empty? && lockfile.nil? && gemfile.nil?
 
         # if a gemfile was provided, but not a lockfile, infer the default lockfile for that gemfile
         lockfile ||= "#{gemfile}.lock" if gemfile
-        # allow short-form lockfile names
         lockfile = expand_lockfile(lockfile)
 
         raise ArgumentError, "Lockfile #{lockfile} is already defined" if lockfile_definitions.key?(lockfile)
@@ -139,9 +144,6 @@ module Bundler
 
         Bundler.ui.debug("Syncing to alternate lockfiles")
 
-        attempts = 1
-        previous_contents = Set.new
-
         default_root = Bundler.root
 
         cache = Cache.new
@@ -158,17 +160,6 @@ module Bundler
 
             relative_lockfile = lockfile_name.relative_path_from(Dir.pwd)
 
-            # prevent infinite loops of tick-tocking back and forth between two versions
-            current_contents = cache.contents(lockfile_name)
-            if previous_contents.include?(current_contents)
-              Bundler.ui.debug("Unable to converge on a single solution for #{lockfile_name}; " \
-                               "perhaps there are conflicting requirements?")
-              attempts = 1
-              previous_contents.clear
-              next
-            end
-            previous_contents << current_contents
-
             # already up to date?
             up_to_date = false
             conflicts = Set.new
@@ -178,11 +169,7 @@ module Bundler
                              checker.deep_check(lockfile_definition, conflicts:)
               end
             end
-            if up_to_date
-              attempts = 1
-              previous_contents.clear
-              next
-            end
+            next if up_to_date
 
             # a missing lockfile that enforces pinned additional dependencies has nothing
             # to be out of sync with, so even a frozen bundle just generates it
@@ -201,7 +188,7 @@ module Bundler
               Bundler.ui.info("Installing gems for #{relative_lockfile}...")
               write_lockfile(lockfile_definition, lockfile_name, cache, install:)
             else
-              Bundler.ui.info("Syncing to #{relative_lockfile}...") if attempts == 1
+              Bundler.ui.info("Syncing to #{relative_lockfile}...")
               synced_any = true
 
               parent_lockfile_name = lockfile_definition[:parent]
@@ -230,6 +217,8 @@ module Bundler
                 TEXT
               end
 
+              added_platforms = []
+              removed_platforms = []
               if lockfile_name.exist?
                 # if the lockfile already exists, "merge" it together
                 parent_lockfile = if adjusted_parent_lockfile_contents == cache.contents(lockfile_name)
@@ -279,8 +268,32 @@ module Bundler
                   parent_spec
                 end
 
-                lockfile.platforms.replace(parent_lockfile.platforms).uniq!
-                lockfile.instance_variable_set(:@ruby_version, parent_lockfile.ruby_version) if lockfile.ruby_version
+                # the parent's versions might depend on gems this lockfile didn't need before.
+                # without them, Bundler considers the lockfile broken, and ignores what's
+                # locked in it, which can update everything
+                names = lockfile.specs.to_set(&:name)
+                pending = lockfile.specs.dup
+                until pending.empty?
+                  pending.shift.dependencies.each do |dependency|
+                    next if names.include?(dependency.name) || !(dependency_specs = parent_specs[dependency.name])
+
+                    names << dependency.name
+                    dependency_changes = true
+                    dependency_specs.each_value do |dependency_spec|
+                      source = dependency_spec.source
+                      lockfile.sources << source unless lockfile.sources.include?(source)
+                      lockfile.specs << dependency_spec
+                      pending << dependency_spec
+                    end
+                  end
+                end
+
+                # leave the lockfile's platforms alone, and let Bundler add and remove them itself
+                # (in write_lockfile), like `bundle lock --add-platform` and `--remove-platform`,
+                # so that it also adds or removes the gems needed for them
+                removed_platforms = lockfile.platforms - parent_lockfile.platforms
+                added_platforms = parent_lockfile.platforms - lockfile.platforms
+                lockfile.instance_variable_set(:@ruby_version, parent_lockfile.ruby_version)
                 unless lockfile.bundler_version == parent_lockfile.bundler_version
                   unlocking_bundler = parent_lockfile.bundler_version
                   lockfile.instance_variable_set(:@bundler_version, parent_lockfile.bundler_version)
@@ -307,24 +320,12 @@ module Bundler
                                                  cache,
                                                  install:,
                                                  dependency_changes:,
-                                                 unlocking_bundler:)
+                                                 unlocking_bundler:,
+                                                 added_platforms:,
+                                                 removed_platforms:)
                 end
               end
               cache.invalidate_lockfile(lockfile_name) if had_changes
-
-              # if we had changes, bundler may have updated some common
-              # dependencies beyond the default lockfile, so re-run it
-              # once to reset them back to the default lockfile's version.
-              # if it's already good, the `check` check at the beginning of
-              # the loop will skip the second sync anyway.
-              if had_changes
-                attempts += 1
-                Bundler.ui.debug("Re-running sync to #{relative_lockfile} to reset common dependencies")
-                redo
-              else
-                attempts = 1
-                previous_contents.clear
-              end
             end
           end
         ensure
@@ -386,10 +387,10 @@ module Bundler
         return [] unless lockfile
         return [lockfile] if recovered
 
-        # we can't tell if Bundler expanded a short name (or "active") to an
-        # absolute path, or if it was written that way, so allow either
+        # we can't tell if Bundler expanded "active" to an absolute path,
+        # or if it was written that way, so allow either
         relative = Pathname.new(lockfile).relative_path_from(Pathname.pwd).to_s
-        relative.include?("/") ? [lockfile] : [lockfile, relative]
+        (relative == "active") ? [lockfile, relative] : [lockfile]
       end
 
       # @!visibility private
@@ -411,6 +412,8 @@ module Bundler
       def reset!
         @lockfile_definitions = {}
         @loaded = false
+        @primary_gemfile = nil
+        @primary_lockfile = nil
       end
 
       # @!visibility private
@@ -418,6 +421,40 @@ module Bundler
         lockfile_definitions[Bundler.default_lockfile(force_original: true)]
       end
 
+      # @!visibility private
+      # The definitions of the other lockfiles, when working with all of them (so not with
+      # BUNDLE_LOCKFILE), for packaging gems from all of them. Their gems need to have been
+      # installed already, by a regular `bundle install`.
+      #
+      # @return [Array<Definition>]
+      def alternate_definitions
+        return [] if lockfile_definitions.empty? || env_lockfile
+
+        # the primary lockfile doesn't have to be declared
+        candidates = lockfile_definitions.values
+        if @primary_lockfile && !lockfile_definitions.key?(@primary_lockfile)
+          candidates += [{ gemfile: @primary_gemfile, lockfile: @primary_lockfile, prepare: nil }]
+        end
+
+        default_root = Bundler.root
+        candidates.filter_map do |lockfile_definition|
+          lockfile = lockfile_definition[:lockfile]
+          # the one Bundler is already working with, which may be an alternate lockfile flagged as active
+          next if lockfile == Bundler.default_lockfile
+          next unless lockfile.exist?
+
+          Bundler.root = lockfile_definition[:gemfile].dirname
+          builder = Dsl.new
+          prepare_block = lockfile_definition[:prepare]
+          builder.eval_gemfile(lockfile_definition[:gemfile], &prepare_block) if prepare_block
+          builder.eval_gemfile(lockfile_definition[:gemfile])
+          # (as the default lockfile, so that it isn't swapped for the active one)
+          Bundler.with_default_lockfile(lockfile) { builder.to_definition(lockfile, {}) }
+        end
+      ensure
+        Bundler.root = default_root if default_root
+      end
+
       private
 
       # @return [Array(String, true or false), nil]
@@ -445,9 +482,6 @@ module Bundler
       end
 
       def expand_lockfile(lockfile)
-        if lockfile.is_a?(String) && !(lockfile.include?("/") || lockfile.end_with?(".lock"))
-          lockfile = "Gemfile.#{lockfile}.lock"
-        end
         # use absolute paths
         lockfile = Bundler.root.join(lockfile).expand_path if lockfile
         # use the default lockfile (Gemfile.lock) if none was given
@@ -504,7 +538,9 @@ module Bundler
                          cache,
                          install:,
                          dependency_changes: false,
-                         unlocking_bundler: false)
+                         unlocking_bundler: false,
+                         added_platforms: [],
+                         removed_platforms: [])
         prepare_block = lockfile_definition[:prepare]
 
         gemfile = lockfile_definition[:gemfile]
@@ -522,6 +558,8 @@ module Bundler
 
         definition = builder.to_definition(lockfile, { bundler: unlocking_bundler })
         definition.instance_variable_set(:@dependency_changes, dependency_changes) if dependency_changes
+        added_platforms.each { |platform| definition.add_platform(platform) }
+        removed_platforms.each { |platform| definition.remove_platform(platform) }
 
         # match the parent lockfile in whether there are checksums
         unless Bundler.frozen_bundle?
@@ -537,6 +575,10 @@ module Bundler
         definition.instance_variable_set(:@lockfile_contents, current_lockfile.read) if current_lockfile.exist?
 
         orig_definition = definition.dup # we might need it twice
+        # `dup` shares the platforms array, and resolving locally changes it (like dropping the ruby platform,
+        # and adding the local platform) before it might fail, which would leave a remote resolution
+        # starting from the wrong platforms
+        orig_definition.instance_variable_set(:@platforms, definition.instance_variable_get(:@platforms).dup)
 
         # install gems for the exact current version of the lockfile
         # this ensures it doesn't re-resolve with only (different)
@@ -572,8 +614,10 @@ module Bundler
           previous_ui_level = Bundler.ui.level
           Bundler.ui.level = "warn"
           begin
-            # force a remote resolution if intermediate gems are missing
-            if definition.instance_variable_get(:@locked_spec_with_missing_deps) ||
+            # force a remote resolution if intermediate gems are missing, or if platforms were added
+            # (a gem's source variant would satisfy them locally, without its platform specific variants)
+            if (added_platforms.any? && !local_only?) ||
+               definition.instance_variable_get(:@locked_spec_with_missing_deps) ||
                definition.instance_variable_get(:@locked_spec_with_invalid_deps) ||
                definition.instance_variable_get(:@missing_lockfile_dep) ||
                definition.instance_variable_get(:@invalid_lockfile_dep)
@@ -593,6 +637,11 @@ module Bundler
               spec.source.specs << spec
             end
             resolve_locally(definition)
+            # a local resolution can only see the gems that are installed, so it can drop platforms
+            # (like ruby) that remote gem variants would keep. the platforms should match the parent's
+            parent_platforms = cache.parser(lockfile_definition[:parent]).platforms.map(&:to_s).sort
+            raise SolveFailure unless local_only? || definition.platforms.map(&:to_s).sort == parent_platforms
+
             missing_checksums = fill_checksums(definition, lockfile_definition[:parent], cache)
             # fetch checksums that aren't available locally
             raise SolveFailure unless missing_checksums.empty? || local_only?
@@ -612,6 +661,11 @@ module Bundler
                   "Run without `--local` to fetch them."
           end
           SharedHelpers.capture_filesystem_access do
+            # a frozen bundle can't change its lockfile, and trying to just fails with a misleading
+            # error from Bundler (like about updating Bundler itself). install what it locks, and
+            # let the check afterwards report what's actually wrong with it
+            next if Bundler.frozen_bundle?
+
             definition.instance_variable_set(:@resolved_bundler_version, unlocking_bundler) if unlocking_bundler
 
             # need to force it to _not_ preserve unknown sections, so that it

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bundler-multilock
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta4
+  version: 2.0.0.beta5
 platform: ruby
 authors:
 - Instructure
@@ -18,7 +18,7 @@ dependencies:
         version: 4.1.0.beta2
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
         version: 4.1.0.beta2
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '4.2'
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -41,6 +41,7 @@ files:
 - lib/bundler/multilock/ext/definition.rb
 - lib/bundler/multilock/ext/dsl.rb
 - lib/bundler/multilock/ext/plugin/dsl.rb
+- lib/bundler/multilock/ext/runtime.rb
 - lib/bundler/multilock/ext/shared_helpers.rb
 - lib/bundler/multilock/ext/source.rb
 - lib/bundler/multilock/ext/source_list.rb