r10k 3.11.0 → 3.14.0

data/r10k.yaml.example

@@ -110,3 +110,31 @@ forge:
   # The 'baseurl' setting indicates where Forge modules should be installed
   # from. This defaults to 'https://forgeapi.puppetlabs.com'
   #baseurl: 'https://forgemirror.example.com'
+
+# Configuration options on how R10k should log its actions
+logging:
+  # The 'level' setting sets the default log level to run R10k actions at.
+  # This value will be overridden by any value set through the command line.
+  #level: warn
+
+  # Specify additional log outputs here, any log4r outputter can be used.
+  # If no log level is specified then the output will use the global level.
+  #outputs:
+  # - type: file
+  #   level: debug
+  #   parameters:
+  #     filename: /var/log/r10k.log
+  #     trunc: true
+  # - type: syslog
+  # - type: email
+  #   only_at: [fatal]
+  #   parameters:
+  #     from: r10k@example.com
+  #     to: sysadmins@example.com
+  #     server: smtp.example.com
+  #     subject: Fatal R10k error occurred
+
+  # The 'disable_default_stderr' setting specifies if the default output on
+  # stderr should be active or not, in case R10k is to be run entirely
+  # through scripts or cronjobs where console output is unwelcome.
+  #disable_default_stderr: false

data/spec/fixtures/unit/action/r10k_logging.yaml

@@ -0,0 +1,12 @@
+---
+  logging:
+    level: FATAL
+
+    outputs:
+      - type: file
+        parameters:
+          filename: r10k.log
+
+      - type: syslog
+
+    disable_default_stderr: true

data/spec/fixtures/unit/puppetfile/forge-override/Puppetfile

@@ -0,0 +1,8 @@
+forge "my.custom.forge.com"
+
+mod "puppetlabs/stdlib", '4.12.0'
+mod "puppetlabs/concat", '2.1.0'
+
+mod 'apache',
+  :git    => 'https://github.com/puppetlabs/puppetlabs-apache',
+  :branch => 'docs_experiment'

data/spec/fixtures/unit/puppetfile/various-modules/Puppetfile

@@ -0,0 +1,10 @@
+mod 'puppetlabs/apt', '2.1.1'
+mod 'puppetlabs/stdlib', :latest
+mod 'puppetlabs/concat'
+mod 'puppetlabs/rpm', '2.1.1-pre1'
+mod 'foo', git: 'this/remote', branch: 'main'
+mod 'bar', git: 'this/remote', tag: 'v1.2.3'
+mod 'baz', git: 'this/remote', commit: '123abc456'
+mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678'
+mod 'buzz', git: 'this/remote', ref: 'refs/heads/main'
+mod 'canary', local: true

data/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new

@@ -0,0 +1,10 @@
+mod 'puppetlabs/apt', '3.0.0'
+mod 'puppetlabs/stdlib', :latest
+mod 'puppetlabs/concat'
+mod 'puppetlabs/rpm', '2.1.1-pre1'
+mod 'foo', git: 'this/remote', branch: 'main'
+mod 'bar', git: 'this/remote', tag: 'v1.2.3'
+mod 'baz', git: 'this/remote', commit: '123abc456'
+mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678'
+mod 'buzz', git: 'this/remote', ref: 'refs/heads/main'
+mod 'canary', local: true

data/spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/managed_symlink_file

@@ -0,0 +1 @@
+../expected_1

data/spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/unmanaged_symlink_dir

@@ -0,0 +1 @@
+../../managed_two

data/spec/fixtures/unit/util/purgeable/managed_one/managed_symlink_dir

@@ -0,0 +1 @@
+spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1

data/spec/fixtures/unit/util/purgeable/managed_one/unmanaged_symlink_file

@@ -0,0 +1 @@
+spec/fixtures/unit/util/purgeable/managed_one/expected_1

data/spec/integration/git/rugged/cache_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'r10k/git/rugged/cache'
+
+describe R10K::Git::Rugged::Cache, :if => R10K::Features.available?(:rugged) do
+  include_context 'Git integration'
+
+  let(:dirname) { 'working-repo' }
+  let(:remote_name) { 'origin' }
+
+  subject { described_class.new(remote) }
+
+  context "syncing with the remote" do
+    before(:each) do
+      subject.reset!
+    end
+
+    describe "with the correct configuration" do
+      it "is able to sync with the remote" do
+        subject.sync
+        expect(subject.synced?).to eq(true)
+      end
+    end
+
+    describe "with a out of date cached remote" do
+      it "updates the cached remote configuration" do
+        subject.repo.update_remote('foo', remote_name)
+        expect(subject.repo.remotes[remote_name]).to eq('foo')
+        subject.sync
+        expect(subject.repo.remotes[remote_name]).to eq(remote)
+      end
+    end
+  end
+end

data/spec/integration/util/purageable_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require 'r10k/util/purgeable'
+require 'r10k/util/cleaner'
+
+require 'tmpdir'
+
+RSpec.describe R10K::Util::Purgeable do
+  it 'purges only unmanaged files' do
+    Dir.mktmpdir do |envdir|
+      managed_directory = "#{envdir}/managed_one"
+      desired_contents = [
+        "#{managed_directory}/expected_1",
+        "#{managed_directory}/managed_subdir_1",
+        "#{managed_directory}/managed_symlink_dir",
+        "#{managed_directory}/managed_subdir_1/subdir_expected_1",
+        "#{managed_directory}/managed_subdir_1/managed_symlink_file",
+      ]
+
+      FileUtils.cp_r('spec/fixtures/unit/util/purgeable/managed_one/',
+                     managed_directory)
+
+      cleaner = R10K::Util::Cleaner.new([managed_directory], desired_contents)
+
+      cleaner.purge!({ recurse: true, whitelist: ["**/subdir_allowlisted_2"] })
+
+      # Files present after purge
+      expect(File.exist?("#{managed_directory}/expected_1")).to be true
+      expect(File.exist?("#{managed_directory}/managed_subdir_1")).to be true
+      expect(File.exist?("#{managed_directory}/managed_symlink_dir")).to be true
+      expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_expected_1")).to be true
+      expect(File.exist?("#{managed_directory}/managed_subdir_1/managed_symlink_file")).to be true
+      expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_allowlisted_2")).to be true
+
+      # Purged files
+      expect(File.exist?("#{managed_directory}/unmanaged_1")).to be false
+      expect(File.exist?("#{managed_directory}/managed_subdir_1/unmanaged_symlink_dir")).to be false
+      expect(File.exist?("#{managed_directory}/unmanaged_symlink_file")).to be false
+      expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_unmanaged_1")).to be false
+    end
+  end
+end

data/spec/r10k-mocks/mock_env.rb

@@ -1,6 +1,9 @@
 require 'r10k/environment'
+require 'r10k/util/purgeable'
 
 class R10K::Environment::Mock < R10K::Environment::Base
+  include R10K::Util::Purgeable
+
   def sync
     "synced"
   end

data/spec/r10k-mocks/mock_source.rb

@@ -5,9 +5,13 @@ class R10K::Source::Mock < R10K::Source::Base
   R10K::Source.register(:mock, self)
 
   def environments
-    corrected_environment_names = @options[:environments].map do |env|
-      R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn')
+    if @_environments.nil?
+      corrected_environment_names = @options[:environments].map do |env|
+        R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn')
+      end
+      @_environments = corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) }
     end
-    corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) }
+
+    @_environments
   end
 end

data/spec/shared-contexts/tarball.rb

@@ -0,0 +1,32 @@
+require 'tmpdir'
+require 'fileutils'
+
+shared_context "Tarball" do
+  # Suggested subject:
+  #
+  #   subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) }
+  #
+  let(:fixture_tarball) do
+    File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT)
+  end
+
+  let(:fixture_checksum) { '292e692ad18faabd4f9b21037d51f0185e04b69f82c522a54af91fb5b88c2d3b' }
+
+  # Use tmpdir for cached tarballs
+  let(:tmpdir) { Dir.mktmpdir }
+
+  # `moduledir` and `cache_root` are available for examples to use in creating
+  # their subjects
+  let(:moduledir) { File.join(tmpdir, 'modules').tap { |path| Dir.mkdir(path) } }
+  let(:cache_root) { File.join(tmpdir, 'cache').tap { |path| Dir.mkdir(path) } }
+
+  around(:each) do |example|
+    if subject.is_a?(R10K::Tarball)
+      subject.settings[:cache_root] = cache_root
+    elsif subject.respond_to?(:tarball) && subject.tarball.is_a?(R10K::Tarball)
+      subject.tarball.settings[:cache_root] = cache_root
+    end
+    example.run
+    FileUtils.remove_entry_secure(tmpdir)
+  end
+end

data/spec/spec_helper.rb

@@ -19,6 +19,7 @@ require 'r10k'
 Dir.glob(File.expand_path('spec/shared-examples/**/*.rb', PROJECT_ROOT)).each { |file| require file }
 
 require 'shared-contexts/git-fixtures'
+require 'shared-contexts/tarball'
 require 'matchers/exit_with'
 require 'matchers/match_realpath'
 require 'r10k-mocks'

data/spec/unit/action/deploy/environment_spec.rb

@@ -63,6 +63,10 @@ describe R10K::Action::Deploy::Environment do
       described_class.new({ :'exclude-spec' => true }, [], {})
     end
 
+    it 'can accept an incremental option' do
+      described_class.new({ :incremental => true }, [], {})
+    end
+
     describe "initializing errors" do
       let (:settings) { { deploy: { purge_levels: [:environment],
                                     purge_whitelist: ['coolfile', 'coolfile2'],
@@ -91,27 +95,70 @@ describe R10K::Action::Deploy::Environment do
 
     describe "with puppetfile or modules flag" do
       let(:deployment) { R10K::Deployment.new(mock_config) }
-      let(:puppetfile) { instance_double("R10K::Puppetfile", modules: []).as_null_object }
+      let(:loader) do
+        instance_double("R10K::ModuleLoader::Puppetfile",
+                        :load => {
+                          :modules => ['foo'],
+                          :purge_exclusions => [],
+                          :managed_directories => [],
+                          :desired_contents => []
+                        }
+                       ).as_null_object
+      end
 
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
-        expect(R10K::Puppetfile).to receive(:new).and_return(puppetfile).at_least(:once)
+        expect(R10K::ModuleLoader::Puppetfile).to receive(:new).
+          and_return(loader).at_least(:once)
       end
 
-      it "syncs the puppetfile when given the puppetfile flag" do
-        expect(puppetfile).to receive(:sync)
+      it "syncs the puppetfile content when given the puppetfile flag" do
+        expect(loader).to receive(:load).exactly(4).times
+        expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times
         action = described_class.new({config: "/some/nonexistent/path", puppetfile: true}, [], {})
         action.call
       end
 
       it "syncs the puppetfile when given the modules flag" do
-        expect(puppetfile).to receive(:sync)
+        expect(loader).to receive(:load).exactly(4).times
+        expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times
         action = described_class.new({config: "/some/nonexistent/path", modules: true}, [], {})
         action.call
       end
+    end
+
+    describe "with incremental flag" do
+      let(:loader) do
+        instance_double("R10K::ModuleLoader::Puppetfile",
+                        :load => {
+                          :modules => ['foo'],
+                          :purge_exclusions => [],
+                          :managed_directories => [],
+                          :desired_contents => []
+                        }
+                       ).as_null_object
+      end
+
+      before do
+        expect(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings|
+          original.call(mock_config.merge(settings))
+        end
+        expect(R10K::ModuleLoader::Puppetfile).to receive(:new).
+          and_return(loader).at_least(:once)
+      end
 
+      it "incremental flag causes the module definitons to be preloaded by the loader" do
+        expect(loader).to receive(:load_metadata).exactly(4).times
+        action = described_class.new({:config => "/some/nonexistent/path",
+                                      :modules => true,
+                                      :incremental => true},
+                                      [],
+                                      {})
+        action.call
+      end
     end
 
+
     describe "with an environment that doesn't exist" do
       let(:deployment) do
         R10K::Deployment.new(mock_config)
@@ -224,19 +271,20 @@ describe R10K::Action::Deploy::Environment do
 
     describe "Purging white/allowlist" do
 
-      let(:settings) { { deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
-      let(:overrides) { { environments: {}, modules: {}, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
+      let(:settings) { { pool_size: 4, deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
+      let(:overrides) { { environments: {}, modules: { pool_size: 4 }, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } }
       let(:deployment) do
-        R10K::Deployment.new(mock_config.merge(overrides))
+        R10K::Deployment.new(mock_config.merge({overrides: overrides}))
       end
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
       end
 
       subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) }
 
       it "reads in the purge_allowlist setting and purges accordingly" do
-        expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
+        expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
         expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2'])
         subject.call
       end
@@ -245,7 +293,7 @@ describe R10K::Action::Deploy::Environment do
         let (:settings) { { deploy: { purge_levels: [:environment], purge_whitelist: ['coolfile', 'coolfile2'] } } }
 
         it "reads in the purge_whitelist setting and still sets it to purge_allowlist and purges accordingly" do
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
           expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2'])
           subject.call
         end
@@ -260,7 +308,8 @@ describe R10K::Action::Deploy::Environment do
             requested_environments: ['PREFIX_first']
           },
           modules: {
-            deploy_modules: true
+            deploy_modules: true,
+            pool_size: 4
           },
           purging: {
             purge_levels: purge_levels
@@ -274,6 +323,7 @@ describe R10K::Action::Deploy::Environment do
 
       before do
         expect(R10K::Deployment).to receive(:new).and_return(deployment)
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
       end
 
       subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) }
@@ -292,12 +342,12 @@ describe R10K::Action::Deploy::Environment do
 
         it "only logs about purging deployment" do
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
-            expect(env.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i)
+            expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/)
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged environments for deployment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged environments for deployment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/)
 
           subject.call
         end
@@ -308,11 +358,11 @@ describe R10K::Action::Deploy::Environment do
 
         it "only logs about purging environment" do
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
-            expect(env.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i)
+            expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/)
             original.call(env)
           end.at_least(:once)
-          expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i)
+          expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/)
 
           subject.call
         end
@@ -325,7 +375,7 @@ describe R10K::Action::Deploy::Environment do
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to receive(:debug).with(/not purging unmanaged content for environment/i)
+          expect(subject.logger).to receive(:debug).with(/Not purging unmanaged content for environment/)
 
           subject.call
         end
@@ -335,15 +385,16 @@ describe R10K::Action::Deploy::Environment do
         let(:purge_levels) { [:puppetfile] }
 
         it "only logs about purging puppetfile" do
+          allow(R10K::ContentSynchronizer).to receive(:concurrent_sync)
           expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block|
             if env.name =~ /first/
-              expect(env.logger).to receive(:debug).with(/purging unmanaged puppetfile content/i)
+              expect(env.logger).to receive(:debug).with(/Purging unmanaged Puppetfile content/)
             end
             original.call(env)
           end.at_least(:once)
 
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i)
-          expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/)
+          expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/)
 
           subject.call
         end
@@ -360,6 +411,11 @@ describe R10K::Action::Deploy::Environment do
                 basedir: '/some/nonexistent/path/control',
                 environments: %w[first second]
               }
+            },
+            overrides: {
+              modules: {
+                pool_size: 4
+              }
             }
           )
         )
@@ -367,9 +423,8 @@ describe R10K::Action::Deploy::Environment do
 
       before do
         allow(R10K::Deployment).to receive(:new).and_return(deployment)
-      end
+        allow_any_instance_of(R10K::Environment::Base).to receive(:purge!)
 
-      before(:each) do
         allow(subject).to receive(:write_environment_info!)
         expect(subject.logger).not_to receive(:error)
       end
@@ -531,7 +586,6 @@ describe R10K::Action::Deploy::Environment do
                       })
     end
     let(:mock_forge_module_1) { double(:name => "their_shiny_module", :properties => { :expected => "2.0.0" }) }
-    let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :modules => [mock_git_module_1, mock_git_module_2, mock_forge_module_1]) }
 
     before(:all) do
       @tmp_path = "./tmp-r10k-test-dir/"
@@ -544,10 +598,8 @@ describe R10K::Action::Deploy::Environment do
     end
 
     it "writes the .r10k-deploy file correctly if all goes well" do
-      allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile)
-
       fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"})
-      allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules)
+      allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1])
       subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true)
 
       file_contents = File.read("#{@tmp_path}/.r10k-deploy.json")
@@ -570,10 +622,8 @@ describe R10K::Action::Deploy::Environment do
     end
 
     it "writes the .r10k-deploy file correctly if there's a failure" do
-      allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile)
-
       fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"})
-      allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules)
+      allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1])
       allow(mock_forge_module_1).to receive(:properties).and_raise(StandardError)
       subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true)