r10k 2.6.9 → 3.0.0

data/integration/tests/i18n/deploy_module_with_unicode_in_file_name.rb

@@ -0,0 +1,64 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+require 'beaker-qa-i18n'
+
+test_name 'Deploy module with unicode file name'
+
+#Init
+master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip
+git_environments_path = '/root/environments'
+last_commit = git_last_commit(master, git_environments_path)
+local_files_root_path = ENV['FILES'] || 'files'
+helloworld_module_path = File.join(local_files_root_path, 'modules', 'helloworld')
+r10k_fqp = get_r10k_fqp(master)
+
+#Manifest
+site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
+site_pp = create_site_pp(master_certname, '  include helloworld')
+
+#Verification
+notify_message_regex = /I am in the production environment/
+
+#Teardown
+teardown do
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+test_i18n_strings(10, [:syntax, :white_space]) do |test_string|
+  #Setup
+  test_file_path = File.join(git_environments_path, "site", 'helloworld', 'manifests', test_string)
+
+  step 'Checkout "production" Branch'
+  git_on(master, 'checkout production', git_environments_path)
+
+  step 'Copy "helloworld" Module to "production" Environment Git Repo'
+  scp_to(master, helloworld_module_path, File.join(git_environments_path, "site", 'helloworld'))
+
+  step 'Add unicode file to helloworld Module'
+  create_remote_file(master, test_file_path, 'test file contents')
+
+  step 'Inject New "site.pp" to the "production" Environment'
+  inject_site_pp(master, site_pp_path, site_pp)
+
+  step 'Push Changes'
+  git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path)
+
+  #Tests
+  step 'Deploy "production" Environment via r10k'
+  on(master, "#{r10k_fqp} deploy environment -v")
+
+  step 'test deployment of Unicode file'
+  deployed_test_file_path = "/etc/puppetlabs/code/environments/production/site/helloworld/manifests/#{test_string}"
+  on(master, "test -f #{deployed_test_file_path}", :accept_all_exit_codes => true) do |result|
+    assert(result.exit_code == 0, "The unicode test file #{test_string} was not deployed by r10k")
+  end
+
+  agents.each do |agent|
+    step "Run Puppet Agent"
+    on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result|
+      assert_no_match(/Error:/, result.stderr, 'Unexpected error was detected!')
+      assert_match(notify_message_regex, result.stdout, 'Expected message not found!')
+    end
+  end
+end

data/integration/tests/purging/does_not_purge_files_on_white_list.rb

@@ -0,0 +1,93 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'RK-257 - C98046 - r10k does not purge files on whitelist'
+
+#Init
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+# initalize file content
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+deploy:
+  purge_levels: ['deployment', 'environment', 'puppetfile']
+  purge_whitelist: ['**/*.pp']
+CONF
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Copy Puppetfile to "production" Environment Git Repo'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", "mod 'puppetlabs-stdlib' \n mod 'puppetlabs-motd'")
+
+step 'Push Changes'
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Deploy production'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'commit a new Puppetfile to production'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "puppetlabs-motd"')
+
+step 'Push Changes'
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'create test pp files'
+do_not_purge = [
+    "/etc/puppetlabs/code/environments/production/environment_level.pp",
+    "/etc/puppetlabs/code/environments/production/site/environment_level.pp"
+].each do |file|
+  create_remote_file(master, file, 'this is a test')
+end
+
+purge = [
+    "/etc/puppetlabs/code/environments/production/environment_level.zz",
+    "/etc/puppetlabs/code/environments/production/site/environment_level.zz"
+].each do |file|
+  create_remote_file(master, file, 'this is a test')
+end
+
+#TEST
+step 'Deploy again and check files'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+purge.each do |file|
+  assert_message = "The file #{file}\n was not purged, it was expected to be"
+  assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 1, assert_message)
+end
+
+do_not_purge.each do |file|
+  assert_message = "The file #{file}\n was purged, it was not expected to be"
+  assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 0, assert_message)
+end

data/integration/tests/purging/invalid_whitelist_types.rb

@@ -0,0 +1,63 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'RK-257 - C98043 - verify default whitelist only accepts strings or array of strings'
+
+#Init
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+
+#invalid content to test
+hash_whitelist = '{:cats => \'cats.txt\'}'
+invalid_array_content_whitelist = '[\'cats.txt\', [:broken]]'
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+# initalize file content
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+[hash_whitelist, invalid_array_content_whitelist].each do |whitelist_content|
+  r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+deploy:
+  purge_whitelist: #{whitelist_content}
+  CONF
+
+  step 'Update the "r10k" Config'
+  create_remote_file(master, r10k_config_path, r10k_conf)
+
+  step 'Deploy r10k, and verify that invalid whitelist content causes error'
+  on(master, "#{r10k_fqp} deploy environment -p", :accept_all_exit_codes => true) do |result|
+    error = /did not find expected node content while parsing a flow node/
+    error_message = 'whitelist content did not generate expected error'
+    expect_failure('RK-263') do
+      assert_no_match(result.stdout, error, error_message)
+    end
+  end
+end

data/integration/tests/user_scenario/basic_workflow/negative/neg_disk_full.rb

@@ -20,6 +20,17 @@ r10k_config_bak_path = "#{r10k_config_path}.bak"
 
 tmpfs_path = '/mnt/tmpfs'
 
+if on(master, 'which python', :acceptable_exit_codes => [0, 1]).exit_code == 1
+    python_bin = 'python3'
+else
+    python_bin = 'python'
+end
+
+file_bucket_path = '/opt/filebucket'
+file_bucket_command_path = File.join(file_bucket_path, 'filebucketapp.py')
+file_bucket_command = "#{python_bin} #{file_bucket_command_path}"
+pattern_file_path = File.join(file_bucket_path, 'psuedo_random_128k.pat')
+
 test_files_path = File.join(git_environments_path, 'test_files')
 
 #In-line files
@@ -62,8 +73,7 @@ on(master, "mount -osize=10m tmpfs #{tmpfs_path} -t tmpfs")
 step 'Create Large Binary File'
 create_remote_file(master, File.join(git_environments_path, '.gitattributes'), '*.file binary')
 on(master, "mkdir -p #{test_files_path}")
-# create a 11 mb file to fill the mount
-on(master, "dd if=/dev/urandom of=#{test_files_path}.test.file bs=1048576 count=11")
+on(master, "#{file_bucket_command} -s 11 -f #{test_files_path}/test.file -d #{pattern_file_path}")
 
 step 'Push Changes'
 git_add_commit_push(master, 'production', 'Add large file.', git_environments_path)

data/integration/tests/user_scenario/basic_workflow/negative/neg_duplicate_module_names.rb

@@ -40,12 +40,5 @@ git_add_commit_push(master, 'production', 'Add modules.', git_environments_path)
 #Tests
 step 'Attempt to Deploy via r10k'
 on(master, "#{r10k_fqp} deploy environment -v -p", :acceptable_exit_codes => [0, 1]) do |result|
-  expect_failure('Expected exit code to be wrong due to RK-101') do
-    assert_equal(1, result.exit_code, "Expected command to indicate error with exit code")
-  end
-
-  expect_failure('Expected module to be deployed twice due to RK-101') do
-    matches = result.stderr.scan(deploy_str)
-    assert_equal(1, matches.size, "Expected motd module to be deployed once, but deployed #{matches.size}) times")
-  end
+  assert_equal(1, result.exit_code, "Expected command to indicate error with exit code")
 end

data/integration/tests/user_scenario/basic_workflow/single_env_10000_files.rb

@@ -21,6 +21,16 @@ helloworld_module_path = File.join(local_files_root_path, 'modules', 'helloworld
 test_files = 'test_files'
 test_files_path = File.join(git_environments_path, test_files)
 
+if on(master, 'which python', :acceptable_exit_codes => [0, 1]).exit_code == 1
+  python_bin = 'python3'
+else
+  python_bin = 'python'
+end
+
+file_bucket_path = '/opt/filebucket'
+file_bucket_command_path = File.join(file_bucket_path, 'filebucketapp.py')
+file_bucket_command = "#{python_bin} #{file_bucket_command_path}"
+
 #Manifest
 site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
 site_pp = create_site_pp(master_certname, '  include helloworld')
@@ -50,8 +60,7 @@ inject_site_pp(master, site_pp_path, site_pp)
 step 'Create 10,000 Files'
 create_remote_file(master, File.join(git_environments_path, '.gitattributes'), '*.file binary')
 on(master, "mkdir -p #{test_files_path}")
-# create 10000 1k files with random text
-on(master, "for n in {1..10000}; do dd if=/dev/urandom of=#{test_files_path}/test$( printf %03d \"$n\" ).file bs=1024 count=1; done")
+on(master, "#{file_bucket_command} -k -c 10000 -p #{test_files_path}/test -u .file")
 
 step 'Create MD5 Checksum of Files'
 on(master, "cd #{test_files_path};md5sum *.file > #{checksum_file_name}")

data/integration/tests/user_scenario/basic_workflow/single_env_large_files.rb

@@ -21,6 +21,17 @@ helloworld_module_path = File.join(local_files_root_path, 'modules', 'helloworld
 test_files = 'test_files'
 test_files_path = File.join(git_environments_path, 'test_files')
 
+if on(master, 'which python', :acceptable_exit_codes => [0, 1]).exit_code == 1
+  python_bin = 'python3'
+else
+  python_bin = 'python'
+end
+
+file_bucket_path = '/opt/filebucket'
+file_bucket_command_path = File.join(file_bucket_path, 'filebucketapp.py')
+pattern_file_path = File.join(file_bucket_path, 'psuedo_random_128k.pat')
+file_bucket_command = "#{python_bin} #{file_bucket_command_path}"
+
 #Manifest
 site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
 site_pp = create_site_pp(master_certname, '  include helloworld')
@@ -50,8 +61,7 @@ inject_site_pp(master, site_pp_path, site_pp)
 step 'Create Large Binary Files'
 create_remote_file(master, File.join(git_environments_path, '.gitattributes'), '*.file binary')
 on(master, "mkdir -p #{test_files_path}")
-# create 10 25 MB files with random characters
-on(master, "for n in {1..10}; do dd if=/dev/urandom of=#{test_files_path}/test$( printf %03d \"$n\" ).file bs=1048576 count=25; done")
+on(master, "#{file_bucket_command} -c 10 -s 25 -p #{test_files_path}/test -u .file -d #{pattern_file_path}")
 
 step 'Create MD5 Checksum of Files'
 on(master, "cd #{test_files_path};md5sum *.file > #{checksum_file_name}")

data/lib/r10k/action/deploy/environment.rb

@@ -109,7 +109,7 @@ module R10K
         end
 
         def visit_puppetfile(puppetfile)
-          puppetfile.load(@opts[:'default-branch-override'])
+          puppetfile.load
 
           yield
 
@@ -125,24 +125,11 @@ module R10K
         end
 
         def write_environment_info!(environment, started_at, success)
-          module_deploys = []
-          begin
-            environment.puppetfile.modules.each do |mod|
-              name = mod.name
-              version = mod.version
-              sha = mod.repo.head rescue nil
-              module_deploys.push({:name => name, :version => version, :sha => sha})
-            end
-          rescue
-            logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}")
-          end
-
           File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
               :deploy_success => success,
-              :module_deploys => module_deploys,
             })
 
             f.puts(JSON.pretty_generate(deploy_info))
@@ -159,7 +146,7 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(puppetfile: :self, cachedir: :self, :'no-force' => :self, :'default-branch-override' => :self)
+          super.merge(puppetfile: :self, cachedir: :self, :'no-force' => :self)
         end
       end
     end

data/lib/r10k/action/puppetfile/check.rb

@@ -8,7 +8,7 @@ module R10K
       class Check < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @path)
+          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
           begin
             pf.load!
             $stderr.puts _("Syntax OK")

data/lib/r10k/action/puppetfile/cri_runner.rb

@@ -3,32 +3,20 @@ require 'r10k/action/cri_runner'
 module R10K
   module Action
     module Puppetfile
-      # Extend the default Cri Runner to use the PUPPETFILE environment
-      # variables.
+      # Extend the default Cri Runner with Puppetfile specific opts
       #
       # @api private
-      # @deprecated The use of these environment variables is deprecated and
-      #   will be removed in 3.0.0.
       class CriRunner < R10K::Action::CriRunner
 
         include R10K::Logging
 
         def handle_opts(opts)
           opts[:root]       ||= wd
-          if env['PUPPETFILE_DIR'] || env['PUPPETFILE']
-            logger.warn _("The use of the PUPPETFILE and PUPPETFILE_DIR environment variables is deprecated.")
-          end
-          opts[:moduledir]  ||= env['PUPPETFILE_DIR']
-          opts[:puppetfile] ||= env['PUPPETFILE']
           super(opts)
         end
 
         private
 
-        def env
-          ENV
-        end
-
         def wd
           Dir.getwd
         end

data/lib/r10k/action/puppetfile/purge.rb

@@ -8,7 +8,7 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @path)
+          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
           pf.load!
           pf.purge!
           true

data/lib/r10k/cli/deploy.rb

@@ -53,7 +53,6 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
           DESCRIPTION
 
           flag :p, :puppetfile, 'Deploy modules from a puppetfile'
-          required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
         end

data/lib/r10k/cli/puppetfile.rb

@@ -44,6 +44,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           name  'check'
           usage 'check'
           summary 'Try and load the Puppetfile to verify the syntax is correct.'
+
+          required nil, :puppetfile, 'Path to Puppetfile'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check)
         end
       end
@@ -56,6 +58,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'purge'
           summary 'Purge unmanaged modules from a Puppetfile managed directory'
 
+          required nil, :moduledir, 'Path to install modules to'
+          required nil, :puppetfile, 'Path to Puppetfile'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge)
         end
       end

data/lib/r10k/environment/base.rb

@@ -24,6 +24,11 @@ class R10K::Environment::Base
   #   @return [R10K::Puppetfile] The puppetfile instance associated with this environment
   attr_reader :puppetfile
 
+  # @!attribute [r] puppetfile_name
+  #   @api public
+  #   @return [String] The puppetfile name (relative)
+  attr_reader :puppetfile_name
+
   # Initialize the given environment.
   #
   # @param name [String] The unique name describing this environment.
@@ -36,11 +41,12 @@ class R10K::Environment::Base
     @basedir = basedir
     @dirname = dirname
     @options = options
+    @puppetfile_name = options[:puppetfile_name]
 
     @full_path = File.join(@basedir, @dirname)
     @path = Pathname.new(File.join(@basedir, @dirname))
 
-    @puppetfile  = R10K::Puppetfile.new(@full_path)
+    @puppetfile  = R10K::Puppetfile.new(@full_path, nil, nil, @puppetfile_name)
     @puppetfile.environment = self
   end