r10k 2.3.1 → 2.4.0

data/integration/Gemfile

@@ -1,3 +1,5 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
 gem 'beaker', '~> 2.7'
+gem 'rototiller', '= 0.1.0'
+gem 'beaker-hostgenerator', '~> 0.7.0'

data/integration/Rakefile

@@ -0,0 +1,41 @@
+require 'rototiller'
+
+desc 'The acceptance tests for r10k, run in the beaker framework'
+rototiller_task :acceptance => [:beaker_hostgenerator] do |t|
+
+  flags = [
+      {:name => '--hosts', :default => 'configs/generated', :override_env => 'BEAKER_HOST'},
+      {:name => '--keyfile', :default => "#{ENV['HOME']}/.ssh/id_rsa-acceptance", :override_env => 'BEAKER_KEYFILE'},
+      {:name => '--load-path', :default => 'lib'},
+      {:name => '--pre-suite', :default => 'pre-suite'},
+      {:name => '--tests', :default => 'tests', :override_env => 'BEAKER_TESTS'},
+      {:name => '--preserve-hosts', :default => 'onfail', :override_env => 'BEAKER_PRESERVE_HOSTS'},
+  ]
+  t.add_flag(*flags)
+
+  t.add_env do |env|
+    env.name = 'BEAKER_PE_DIR'
+    env.message = 'The location to download PE from example "http://neptune.puppetlabs.lan/2016.2/ci-ready"'
+  end
+
+  t.add_env do |env|
+    env.name = 'GIT_PROVIDER'
+    env.message = 'The git provider that r10k should use on a SUT'
+  end
+
+  t.add_command({:name => 'beaker --debug', :override_env => 'BEAKER_EXECUTABLE'})
+end
+
+desc 'Generate a host configuration used by Beaker'
+rototiller_task :beaker_hostgenerator do |t|
+
+  t.add_command do |c|
+    c.name = 'beaker-hostgenerator'
+    c.argument = '> configs/generated'
+  end
+
+  #this is a hack :(
+  t.add_flag(:name => '', :default => 'centos6-64mdca-64.fa', :override_env => 'TEST_TARGET')
+
+  t.add_flag(:name => '--global-config', :default => '{forge_host=forge-aio01-petest.puppetlabs.com}', :override_env => 'BHG_GLOBAL_CONFIG')
+end

data/integration/tests/purging/content_not_purged_at_root.rb

@@ -0,0 +1,89 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'RK-256 - C98049 - verify if non-module content is at root of dir, does not cause erroneous purging'
+
+#Init
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+code_dir = '/etc/puppetlabs/code/environments/production'
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+CONF
+
+puppetfile = <<-EOS
+mod 'non_module_object_1',
+  :install_path => './',
+  :git => 'https://github.com/puppetlabs/control-repo.git',
+  :branch => 'production'
+
+mod 'non_module_object_2',
+ :install_path => '',
+ :git => 'https://github.com/puppetlabs/control-repo.git',
+  :branch => 'production'
+EOS
+
+puppetfile_2 = <<-EOS
+mod 'puppetlabs-motd'
+EOS
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Ask r10k to deploy'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Add puppetfile with non-module content at top of directory'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", puppetfile)
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Deploy r10k'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Add puppetfile #2'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", puppetfile_2)
+git_add_commit_push(master, 'production', 'add Puppetfile to production', git_environments_path)
+
+step 'Deploy r10k after adding puppetfile #2'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Verify that non-module object 1 has not been purged'
+on(master, "test -d #{code_dir}/non_module_object_1", :accept_all_exit_codes => true) do |result|
+  non_module_error = 'Non-module object was purged; should have been left alone'
+  assert(result.exit_code == 0, non_module_error)
+end
+
+step 'Verify that non-module object 2 has not been purged'
+on(master, "test -d #{code_dir}/non_module_object_2", :accept_all_exit_codes => true) do |result|
+  non_module_error = 'Non-module object was purged; should have been left alone'
+  assert(result.exit_code == 0, non_module_error)
+end

data/integration/tests/purging/default_purging.rb

@@ -0,0 +1,125 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'RK-256 - C98013 - verify default purging behavior'
+
+#Init
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+CONF
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+# initialize path name a....d (where it is)
+code_dir_path = '/etc/puppetlabs/code'
+fake_environment_path_a = [code_dir_path, 'environments'].join('/')
+fake_environment_path_b = [code_dir_path,'environments', 'production'].join('/')
+fake_environment_path_c = [code_dir_path, 'environments', 'production', 'modules'].join('/')
+fake_environment_path_d = [code_dir_path, 'environments', 'production', 'modules', 'motd'].join('/')  #not sure if need
+
+# initalize directory name a...c (where it is, what it is called)
+fake_dir_a_to_be_purged     = "#{fake_environment_path_a}/fakedir1"
+fake_dir_b_to_be_left_alone = "#{fake_environment_path_b}/fakedir2"
+fake_dir_c_to_be_purged     = "#{fake_environment_path_c}/fakedir3"
+
+# initalize file name a...c (where it is, what it is called)
+fake_file_a_to_be_purged     = "#{fake_environment_path_a}/fakefile1.txt"
+fake_file_b_to_be_left_alone = "#{fake_environment_path_b}/fakefile2.txt"
+fake_file_c_to_be_purged     = "#{fake_environment_path_c}/fakefile3.txt"
+
+# initalize file content
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Create content to be managed by r10k'
+git_on(master, 'checkout production', git_environments_path)
+create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "puppetlabs-motd"')
+git_add_commit_push(master, 'production', 'add Puppetfile to production', git_environments_path)
+
+git_on(master, 'checkout production', git_environments_path)
+git_on(master, 'checkout -b cats', git_environments_path)
+create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "puppetlabs-inifile"')
+git_add_commit_push(master, 'cats', 'add Puppetfile to cats', git_environments_path)
+
+step 'Ask r10k to deploy'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Create fake file and directory at deployment level to be purged'
+create_remote_file(master, fake_file_a_to_be_purged, "foobar nonsense")
+on(master, "mkdir #{fake_dir_a_to_be_purged}")
+
+step 'Create fake file and directory at environment level to be left alone'
+create_remote_file(master, fake_file_b_to_be_left_alone, "foobar nonsense")
+on(master, "mkdir #{fake_dir_b_to_be_left_alone}")
+
+step 'Create fake file and directory at puppetfile level to be purged '
+create_remote_file(master, fake_file_c_to_be_purged, "foobar nonsense")
+on(master, "mkdir #{fake_dir_c_to_be_purged}")
+
+step('Deploy r10k')
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step('Assert to see if deployment level file is not there')
+on(master, "test -f #{fake_file_a_to_be_purged}", :accept_all_exit_codes => true) do |result|
+  file_a_error = 'Puppet file purging was not observed'
+  assert(result.exit_code == 1, file_a_error)
+end
+
+step('Assert to see if deployment level directory is not there')
+on(master, "test -d #{fake_dir_a_to_be_purged}", :accept_all_exit_codes => true) do |result|
+  dir_a_error = 'Puppet directory purging was not observed'
+  assert(result.exit_code == 1, dir_a_error)
+end
+
+step('Assert to see if environment level file is still there after second deployment')
+on(master, "test -f #{fake_file_b_to_be_left_alone}", :accept_all_exit_codes => true) do |result|
+  file_b_error = 'Puppet file purging deleted this file when it should have left it alone :('
+  assert(result.exit_code == 0, file_b_error)
+end
+
+step('Assert to see if environment level directory is still there after second deployment')
+on(master, "test -d #{fake_dir_b_to_be_left_alone}", :accept_all_exit_codes => true) do |result|
+  dir_b_error = 'Puppet directory purging deleted this directory when it should have left it alone :('
+  assert(result.exit_code == 0, dir_b_error)
+end
+
+step('Assert to see if puppetfile level file is not there')
+on(master, "test -f #{fake_file_c_to_be_purged}", :accept_all_exit_codes => true) do |result|
+  file_c_error = 'Puppet file purging was not observed'
+  assert(result.exit_code == 1, file_c_error)
+end
+
+step('Assert to see if puppetfile level directory is not there')
+on(master, "test -d #{fake_dir_c_to_be_purged}", :accept_all_exit_codes => true) do |result|
+  dir_c_error = 'Puppet directory purging was not observed'
+  assert(result.exit_code == 1, dir_c_error)
+end

data/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb

@@ -19,7 +19,7 @@ PUPPETFILE
 puppet_file_path = File.join(git_environments_path, 'Puppetfile')
 
 #Verification
-error_message_regex = /ERROR.*Unable to sync repo to unresolvable ref/
+error_message_regex = /ERROR.*Could not resolve desired ref/
 
 #Teardown
 teardown do

data/lib/r10k.rb

@@ -1,4 +1,11 @@
-module R10K; end
+require 'gettext-setup'
+
+module R10K
+  GettextSetup.initialize(File.absolute_path('../locales', File.dirname(__FILE__)))
+
+  # Attempt to set the R10k error and log message locale
+  FastGettext.locale = ENV["LANG"]
+end
 
 require 'r10k/version'
 require 'r10k/logging'

data/lib/r10k/action/deploy/deploy_helpers.rb

@@ -9,7 +9,7 @@ module R10K
         # @raise [SystemExit] If no config file was loaded
         def expect_config!
           if @config.nil?
-            logger.fatal("No configuration file given, no config file found in current directory, and no global config present")
+            logger.fatal(_("No configuration file given, no config file found in current directory, and no global config present"))
             exit(8)
           end
         end
@@ -23,8 +23,8 @@ module R10K
         def check_write_lock!(config)
           write_lock = config.fetch(:deploy, {})[:write_lock]
           if write_lock
-            logger.fatal("Making changes to deployed environments has been administratively disabled.")
-            logger.fatal("Reason: #{write_lock}")
+            logger.fatal(_("Making changes to deployed environments has been administratively disabled."))
+            logger.fatal(_("Reason: %{write_lock}") % {write_lock: write_lock})
             exit(16)
           end
         end

data/lib/r10k/action/deploy/environment.rb

@@ -13,9 +13,13 @@ module R10K
 
         include R10K::Action::Deploy::DeployHelpers
 
-        def initialize(opts, argv, settings = {})
-          @purge = true
+        def initialize(opts, argv, settings = nil)
+          settings ||= {}
+          @purge_levels = settings.fetch(:deploy, {}).fetch(:purge_levels, [])
+          @user_purge_whitelist = settings.fetch(:deploy, {}).fetch(:purge_whitelist, [])
+
           super
+
           @argv = @argv.map { |arg| arg.gsub(/\W/,'_') }
         end
 
@@ -45,13 +49,15 @@ module R10K
           undeployable = undeployable_environment_names(deployment.environments, @argv)
           if !undeployable.empty?
             @visit_ok = false
-            logger.error "Environment(s) \'#{undeployable.join(", ")}\' cannot be found in any source and will not be deployed."
+            logger.error _("Environment(s) \'%{environments}\' cannot be found in any source and will not be deployed.") % {environments: undeployable.join(", ")}
           end
 
           yield
 
-          deployment.purge! if @purge
-
+          if @purge_levels.include?(:deployment)
+            logger.debug("Purging unmanaged environments for deployment...")
+            deployment.purge!
+          end
         ensure
           if (postcmd = @settings[:postrun])
             subproc = R10K::Util::Subprocess.new(postcmd)
@@ -66,45 +72,60 @@ module R10K
 
         def visit_environment(environment)
           if !(@argv.empty? || @argv.any? { |name| environment.dirname == name })
-            logger.debug1("Environment #{environment.dirname} does not match environment name filter, skipping")
+            logger.debug1(_("Environment %{env_dir} does not match environment name filter, skipping") % {env_dir: environment.dirname})
             return
           end
 
           started_at = Time.new
 
           status = environment.status
-          logger.info "Deploying environment #{environment.path}"
+          logger.info _("Deploying environment %{env_path}") % {env_path: environment.path}
 
           environment.sync
-          logger.info "Environment #{environment.dirname} is now at #{environment.signature}"
+          logger.info _("Environment %{env_dir} is now at %{env_signature}") % {env_dir: environment.dirname, env_signature: environment.signature}
 
           if status == :absent || @puppetfile
             if status == :absent
-              logger.debug("Environment #{environment.dirname} is new, updating all modules")
+              logger.debug(_("Environment %{env_dir} is new, updating all modules") % {env_dir: environment.dirname})
             end
 
             yield
           end
 
-          write_environment_info!(environment, started_at)
+          if @purge_levels.include?(:environment)
+            if @visit_ok
+              logger.debug("Purging unmanaged content for environment '#{environment.dirname}'...")
+              environment.purge!(:recurse => true, :whitelist => environment.whitelist(@user_purge_whitelist))
+            else
+              logger.debug("Not purging unmanaged content for environment '#{environment.dirname}' due to prior deploy failures.")
+            end
+          end
+
+          write_environment_info!(environment, started_at, @visit_ok)
         end
 
         def visit_puppetfile(puppetfile)
           puppetfile.load
+
           yield
-          puppetfile.purge!
+
+          if @purge_levels.include?(:puppetfile)
+            logger.debug("Purging unmanaged Puppetfile content for environment '#{puppetfile.environment.dirname}'...")
+            puppetfile.purge!
+          end
         end
 
         def visit_module(mod)
-          logger.info "Deploying module #{mod.path}"
+          logger.info _("Deploying Puppetfile content %{mod_path}") % {mod_path: mod.path}
           mod.sync
         end
 
-        def write_environment_info!(environment, started_at)
+        def write_environment_info!(environment, started_at, success)
           File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
+              :deploy_success => success,
             })
 
             f.puts(JSON.pretty_generate(deploy_info))
@@ -121,7 +142,7 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(puppetfile: :self, cachedir: :self, purge: true)
+          super.merge(puppetfile: :self, cachedir: :self)
         end
       end
     end

data/lib/r10k/action/deploy/module.rb

@@ -35,9 +35,9 @@ module R10K
 
         def visit_environment(environment)
           if @opts[:environment] && (@opts[:environment] != environment.dirname)
-            logger.debug1("Only updating modules in environment #{@opts[:environment]}, skipping environment #{environment.path}")
+            logger.debug1(_("Only updating modules in environment %{opt_env} skipping environment %{env_path}") % {opt_env: @opts[:environment], env_path: environment.path})
           else
-            logger.debug1("Updating modules #{@argv.inspect} in environment #{environment.path}")
+            logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @argv.inspect, env_path: environment_path})
             yield
           end
         end
@@ -49,10 +49,10 @@ module R10K
 
         def visit_module(mod)
           if @argv.include?(mod.name)
-            logger.info "Deploying module #{mod.path}"
+            logger.info _("Deploying module %{mod_path}") % {mod_path: mod.path}
             mod.sync
           else
-            logger.debug1("Only updating modules #{@argv.inspect}, skipping module #{mod.name}")
+            logger.debug1(_("Only updating modules %{modules}, skipping module %{mod_name}") % {modules: @argv.inspect, mod_name: mod.name})
           end
         end
 

data/lib/r10k/action/puppetfile/check.rb

@@ -11,7 +11,7 @@ module R10K
           pf = R10K::Puppetfile.new(@root, @moduledir, @path)
           begin
             pf.load!
-            $stderr.puts "Syntax OK"
+            $stderr.puts _("Syntax OK")
             true
           rescue => e
             $stderr.puts R10K::Errors::Formatting.format_exception(e, @trace)