r10k 3.11.0 → 3.12.0

data/spec/fixtures/unit/puppetfile/forge-override/Puppetfile

@@ -0,0 +1,8 @@
+forge "my.custom.forge.com"
+
+mod "puppetlabs/stdlib", '4.12.0'
+mod "puppetlabs/concat", '2.1.0'
+
+mod 'apache',
+  :git    => 'https://github.com/puppetlabs/puppetlabs-apache',
+  :branch => 'docs_experiment'

data/spec/fixtures/unit/puppetfile/various-modules/Puppetfile

@@ -0,0 +1,9 @@
+mod 'puppetlabs/apt', '2.1.1'
+mod 'puppetlabs/stdlib', :latest
+mod 'puppetlabs/concat'
+mod 'puppetlabs/rpm', '2.1.1-pre1'
+mod 'foo', git: 'this/remote', branch: 'main'
+mod 'bar', git: 'this/remote', tag: 'v1.2.3'
+mod 'baz', git: 'this/remote', commit: '123abc456'
+mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678'
+mod 'buzz', git: 'this/remote', ref: 'refs/heads/main'

data/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new

@@ -0,0 +1,9 @@
+mod 'puppetlabs/apt', '3.0.0'
+mod 'puppetlabs/stdlib', :latest
+mod 'puppetlabs/concat'
+mod 'puppetlabs/rpm', '2.1.1-pre1'
+mod 'foo', git: 'this/remote', branch: 'main'
+mod 'bar', git: 'this/remote', tag: 'v1.2.3'
+mod 'baz', git: 'this/remote', commit: '123abc456'
+mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678'
+mod 'buzz', git: 'this/remote', ref: 'refs/heads/main'

data/spec/r10k-mocks/mock_env.rb

@@ -1,6 +1,9 @@
 require 'r10k/environment'
+require 'r10k/util/purgeable'
 
 class R10K::Environment::Mock < R10K::Environment::Base
+  include R10K::Util::Purgeable
+
   def sync
     "synced"
   end

data/spec/r10k-mocks/mock_source.rb

@@ -5,9 +5,13 @@ class R10K::Source::Mock < R10K::Source::Base
   R10K::Source.register(:mock, self)
 
   def environments
-    corrected_environment_names = @options[:environments].map do |env|
-      R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn')
+    if @_environments.nil?
+      corrected_environment_names = @options[:environments].map do |env|
+        R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn')
+      end
+      @_environments = corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) }
     end
-    corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) }
+
+    @_environments
   end
 end

data/spec/unit/action/deploy/environment_spec.rb

@@ -63,6 +63,10 @@ describe R10K::Action::Deploy::Environment do
       described_class.new({ :'exclude-spec' => true }, [], {})
     end
 
+    it 'can accept an incremental option' do
+      described_class.new({ :incremental => true }, [], {})
+    end
+
     describe "initializing errors" do
       let (:settings) { { deploy: { purge_levels: [:environment],
                                     purge_whitelist: ['coolfile', 'coolfile2'],
@@ -91,27 +95,70 @@ describe R10K::Action::Deploy::Environment do
 
     describe "with puppetfile or modules flag" do
       let(:deployment) { R10K::Deployment.new(mock_config) }
-      let(:puppetfile) { instance_double("R10K::Puppetfile", modules: []).as_null_object }
+      let(:loader) do
+        instance_double("R10K::ModuleLoader::Puppetfile",
+                        :load => {
+                          :modules => ['foo'],
+                          :purge_exclusions => [],
+                          :managed_directories => [],
+                          :desired_contents => []
+                        }
+                       ).as_null_object
+      end
 
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
-        expect(R10K::Puppetfile).to receive(:new).and_return(puppetfile).at_least(:once)
+        expect(R10K::ModuleLoader::Puppetfile).to receive(:new).
+          and_return(loader).at_least(:once)
       end
 
-      it "syncs the puppetfile when given the puppetfile flag" do
-        expect(puppetfile).to receive(:sync)
+      it "syncs the puppetfile content when given the puppetfile flag" do
+        expect(loader).to receive(:load).exactly(4).times
+        expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times
         action = described_class.new({config: "/some/nonexistent/path", puppetfile: true}, [], {})
         action.call
       end
 
       it "syncs the puppetfile when given the modules flag" do
-        expect(puppetfile).to receive(:sync)
+        expect(loader).to receive(:load).exactly(4).times
+        expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times
         action = described_class.new({config: "/some/nonexistent/path", modules: true}, [], {})
         action.call
       end
+    end
+
+    describe "with incremental flag" do
+      let(:loader) do
+        instance_double("R10K::ModuleLoader::Puppetfile",
+                        :load => {
+                          :modules => ['foo'],
+                          :purge_exclusions => [],
+                          :managed_directories => [],
+                          :desired_contents => []
+                        }
+                       ).as_null_object
+      end
+
+      before do
+        expect(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings|
+          original.call(mock_config.merge(settings))
+        end
+        expect(R10K::ModuleLoader::Puppetfile).to receive(:new).
+          and_return(loader).at_least(:once)
+      end
 
+      it "incremental flag causes the module definitons to be preloaded by the loader" do
+        expect(loader).to receive(:load_metadata).exactly(4).times
+        action = described_class.new({:config => "/some/nonexistent/path",
+                                      :modules => true,
+                                      :incremental => true},
+                                      [],
+                                      {})
+        action.call
+      end
     end
 
+
     describe "with an environment that doesn't exist" do
       let(:deployment) do
         R10K::Deployment.new(mock_config)
@@ -224,19 +271,20 @@ describe R10K::Action::Deploy::Environment do
 
     describe "Purging white/allowlist" do
 
-      let(:settings) { { deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
-      let(:overrides) { { environments: {}, modules: {}, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
+      let(:settings) { { pool_size: 4, deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
+      let(:overrides) { { environments: {}, modules: { pool_size: 4 }, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
       let(:deployment) do
-        R10K::Deployment.new(mock_config.merge(overrides))
+        R10K::Deployment.new(mock_config.merge({overrides: overrides}))
       end
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
       end
 
       subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) }
 
       it "reads in the purge_allowlist setting and purges accordingly" do
-        expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
+        expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
         expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2'])
         subject.call
       end
@@ -245,7 +293,7 @@ describe R10K::Action::Deploy::Environment do
         let (:settings) { { deploy: { purge_levels: [:environment], purge_whitelist: ['coolfile', 'coolfile2'] } } }
 
         it "reads in the purge_whitelist setting and still sets it to purge_allowlist and purges accordingly" do
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
           expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2'])
           subject.call
         end
@@ -260,7 +308,8 @@ describe R10K::Action::Deploy::Environment do
             requested_environments: ['PREFIX_first']
           },
           modules: {
-            deploy_modules: true
+            deploy_modules: true,
+            pool_size: 4
           },
           purging: {
             purge_levels: purge_levels
@@ -274,6 +323,7 @@ describe R10K::Action::Deploy::Environment do
 
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
       end
 
       subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) }
@@ -292,12 +342,12 @@ describe R10K::Action::Deploy::Environment do
 
         it "only logs about purging deployment" do
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
-            expect(env.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i)
+            expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/)
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged environments for deployment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged environments for deployment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/)
 
           subject.call
         end
@@ -308,11 +358,11 @@ describe R10K::Action::Deploy::Environment do
 
         it "only logs about purging environment" do
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
-            expect(env.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i)
+            expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/)
             original.call(env)
           end.at_least(:once)
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/)
 
           subject.call
         end
@@ -325,7 +375,7 @@ describe R10K::Action::Deploy::Environment do
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to receive(:debug).with(/not purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Not purging unmanaged content for environment/)
 
           subject.call
         end
@@ -335,15 +385,16 @@ describe R10K::Action::Deploy::Environment do
         let(:purge_levels) { [:puppetfile] }
 
         it "only logs about purging puppetfile" do
+          allow(R10K::ContentSynchronizer).to receive(:concurrent_sync)
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
             if env.name =~ /first/
-              expect(env.logger).to receive(:debug).with(/purging unmanaged puppetfile content/i)
+              expect(env.logger).to receive(:debug).with(/Purging unmanaged Puppetfile content/)
             end
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/)
 
           subject.call
         end
@@ -360,6 +411,11 @@ describe R10K::Action::Deploy::Environment do
                 basedir: '/some/nonexistent/path/control',
                 environments: %w[first second]
               }
+            },
+            overrides: {
+              modules: {
+                pool_size: 4
+              }
             }
           )
         )
@@ -367,9 +423,8 @@ describe R10K::Action::Deploy::Environment do
 
       before do
         allow(R10K::Deployment).to receive(:new).and_return(deployment)
-      end
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
 
-      before(:each) do
         allow(subject).to receive(:write_environment_info!)
         expect(subject.logger).not_to receive(:error)
       end
@@ -531,7 +586,6 @@ describe R10K::Action::Deploy::Environment do
                       })
     end
     let(:mock_forge_module_1) { double(:name => "their_shiny_module", :properties => { :expected => "2.0.0" }) }
-    let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :modules => [mock_git_module_1, mock_git_module_2, mock_forge_module_1]) }
 
     before(:all) do
       @tmp_path = "./tmp-r10k-test-dir/"
@@ -544,10 +598,8 @@ describe R10K::Action::Deploy::Environment do
     end
 
     it "writes the .r10k-deploy file correctly if all goes well" do
-      allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile)
-
       fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"})
-      allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules)
+      allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1])
       subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true)
 
       file_contents = File.read("#{@tmp_path}/.r10k-deploy.json")
@@ -570,10 +622,8 @@ describe R10K::Action::Deploy::Environment do
     end
 
     it "writes the .r10k-deploy file correctly if there's a failure" do
-      allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile)
-
       fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"})
-      allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules)
+      allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1])
       allow(mock_forge_module_1).to receive(:properties).and_raise(StandardError)
       subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true)
 

data/spec/unit/action/deploy/module_spec.rb

@@ -77,6 +77,11 @@ describe R10K::Action::Deploy::Module do
             basedir: '/some/nonexistent/path/control',
             environments: %w[first second]
           }
+        },
+        overrides: {
+          modules: {
+            pool_size: 4
+          }
         }
       )
     end
@@ -97,28 +102,22 @@ describe R10K::Action::Deploy::Module do
         )
       end
 
-      before do
-        @modules = []
+      it 'generate_types is true' do
+        expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true)
+      end
+
+      it 'only calls puppet generate types on environments where the specified module was updated' do
         allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-          mod = R10K::Module::Local.new(environment.name, '/fakedir', {}, environment)
-          if mod.name == 'first'
+          if environment.name == 'first'
+            expect(environment).to receive(:deploy).and_return(['first'])
             expect(environment).to receive(:generate_types!)
           else
+            expect(environment).to receive(:deploy).and_return([])
             expect(environment).not_to receive(:generate_types!)
           end
-          @modules << mod
-          expect(environment.puppetfile).to receive(:modules).and_return([mod]).twice
           original.call(environment, &block)
         end
-      end
-
-      it 'generate_types is true' do
-        expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true)
-      end
-
-      it 'only calls puppet generate types on environments with specified module' do
         subject.call
-        expect(@modules.length).to be(2)
       end
     end
 
@@ -248,29 +247,27 @@ describe R10K::Action::Deploy::Module do
         # For this test we want to have realistic Modules and access to
         # their internal Repos to validate the sync. Unfortunately, to
         # do so we do some invasive mocking, effectively implementing
-        # our own R10K::Puppetfile#load. We directly update the Puppetfile's
-        # internal ModuleLoader and then call `load` on it so it will create
-        # the correct loaded_content.
-        puppetfile = environment.puppetfile
-        loader = puppetfile.loader
-        expect(puppetfile).to receive(:load) do
+        # our own R10K::ModuleLoader::Puppetfile#load. We directly update
+        # the Environment's internal ModuleLoader and then call `load` on
+        # it so it will create the correct loaded_content.
+        loader = environment.loader
+        allow(loader).to receive(:puppetfile_content).and_return('')
+        expect(loader).to receive(:load) do
           loader.add_module('mod1', { git: 'git://remote' })
           loader.add_module('mod2', { git: 'git://remote' })
           loader.add_module('mod3', { git: 'git://remote' })
 
-          allow(loader).to receive(:puppetfile_content).and_return('')
-          loaded_content = loader.load
-          puppetfile.instance_variable_set(:@loaded_content, loaded_content)
-          puppetfile.instance_variable_set(:@loaded, true)
-        end
-
-        puppetfile.modules.each do |mod|
-          if ['mod1', 'mod2'].include?(mod.name)
-            expect(mod.should_sync?).to be(true)
-          else
-            expect(mod.should_sync?).to be(false)
+          loaded_content = loader.load!
+          loaded_content[:modules].each do |mod|
+            if ['mod1', 'mod2'].include?(mod.name)
+              expect(mod.should_sync?).to be(true)
+            else
+              expect(mod.should_sync?).to be(false)
+            end
+            expect(mod).to receive(:sync).and_call_original
           end
-          expect(mod).to receive(:sync).and_call_original
+
+          loaded_content
         end
 
         original.call(environment, &block)
@@ -307,36 +304,35 @@ describe R10K::Action::Deploy::Module do
                                                                                      R10K::Environment::Name.new('second', {})])
 
       expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-        puppetfile = environment.puppetfile
+        loader = environment.loader
 
         if environment.name == 'first'
           # For this test we want to have realistic Modules and access to
           # their internal Repos to validate the sync. Unfortunately, to
           # do so we do some invasive mocking, effectively implementing
-          # our own R10K::Puppetfile#load. We directly update the Puppetfile's
-          # internal ModuleLoader and then call `load` on it so it will create
-          # the correct loaded_content.
-          loader = puppetfile.loader
-          expect(puppetfile).to receive(:load) do
+          # our own R10K::ModuleLoader::Puppetfile#load. We directly update
+          # the Environment's internal ModuleLoader and then call `load` on
+          # it so it will create the correct loaded_content.
+          allow(loader).to receive(:puppetfile_content).and_return('')
+          expect(loader).to receive(:load) do
             loader.add_module('mod1', { git: 'git://remote' })
             loader.add_module('mod2', { git: 'git://remote' })
 
-            allow(loader).to receive(:puppetfile_content).and_return('')
-            loaded_content = loader.load
-            puppetfile.instance_variable_set(:@loaded_content, loaded_content)
-            puppetfile.instance_variable_set(:@loaded, true)
-          end
-
-          puppetfile.modules.each do |mod|
-            if mod.name == 'mod1'
-              expect(mod.should_sync?).to be(true)
-            else
-              expect(mod.should_sync?).to be(false)
+            loaded_content = loader.load!
+            loaded_content[:modules].each do |mod|
+              if mod.name == 'mod1'
+                expect(mod.should_sync?).to be(true)
+              else
+                expect(mod.should_sync?).to be(false)
+              end
+              expect(mod).to receive(:sync).and_call_original
             end
-            expect(mod).to receive(:sync).and_call_original
+
+            loaded_content
           end
+
         else
-          expect(puppetfile).not_to receive(:load)
+          expect(loader).not_to receive(:load)
         end
 
         original.call(environment, &block)
@@ -415,16 +411,13 @@ describe R10K::Action::Deploy::Module do
           allow(mock_subprocess).to receive(:logger=)
           expect(mock_subprocess).to receive(:execute)
 
-          mock_mod = double('mock_mod', name: 'mod1')
-
           expect(R10K::Util::Subprocess).to receive(:new).
             with(["/generate/types/wrapper", "first"]).
             and_return(mock_subprocess)
 
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
             if environment.name == 'first'
-              expect(environment).to receive(:deploy).and_return(true)
-              expect(environment).to receive(:modules).and_return([mock_mod])
+              expect(environment).to receive(:deploy).and_return(['first'])
             end
             original.call(environment, &block)
           end.exactly(3).times
@@ -455,12 +448,9 @@ describe R10K::Action::Deploy::Module do
             with(["/generate/types/wrapper", "first third"]).
             and_return(mock_subprocess)
 
-          mock_mod = double('mock_mod', name: 'mod1')
-
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-            expect(environment).to receive(:deploy).and_return(true)
             if ['first', 'third'].include?(environment.name)
-              expect(environment).to receive(:modules).and_return([mock_mod])
+              expect(environment).to receive(:deploy).and_return(['mod1'])
             end
             original.call(environment, &block)
           end.exactly(3).times
@@ -473,9 +463,7 @@ describe R10K::Action::Deploy::Module do
 
           mock_mod2 = double('mock_mod', name: 'mod2')
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-            expect(environment).to receive(:deploy).and_return(true)
-            # Envs have a different module than the one we asked to deploy
-            expect(environment).to receive(:modules).and_return([mock_mod2])
+            expect(environment).to receive(:deploy).and_return([])
             original.call(environment, &block)
           end.exactly(3).times