puppetlabs_spec_helper 2.13.1 → 3.0.0

data/lib/puppetlabs_spec_helper/tasks/beaker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rspec/core/rake_task'
 
 module PuppetlabsSpecHelper; end
@@ -10,9 +12,7 @@ module PuppetlabsSpecHelper::Tasks::BeakerHelpers
 
   # cache the repositories and return a hash object
   def repositories
-    unless @repositories
-      @repositories = fixtures('repositories')
-    end
+    @repositories ||= fixtures('repositories')
     @repositories
   end
 
@@ -20,6 +20,7 @@ module PuppetlabsSpecHelper::Tasks::BeakerHelpers
   # @return [Array<String>]
   def beaker_node_sets
     return @beaker_nodes if @beaker_nodes
+
     @beaker_nodes = Dir['spec/acceptance/nodesets/*.yml'].sort.map do |node_set|
       node_set.slice!('.yml')
       File.basename(node_set)
@@ -50,7 +51,7 @@ module PuppetlabsSpecHelper::Tasks::BeakerHelpers
     end
   end
 end
-include PuppetlabsSpecHelper::Tasks::BeakerHelpers
+include PuppetlabsSpecHelper::Tasks::BeakerHelpers # legacy support code # rubocop:disable Style/MixinUsage
 
 desc 'Run beaker acceptance tests'
 RSpec::Core::RakeTask.new(:beaker) do |t|
@@ -58,24 +59,26 @@ RSpec::Core::RakeTask.new(:beaker) do |t|
 end
 
 class SetupBeaker
-  def self.setup_beaker(t)
-    t.rspec_opts = ['--color']
-    t.pattern = 'spec/acceptance'
+  def self.setup_beaker(task)
+    task.rspec_opts = []
+    task.pattern = 'spec/acceptance'
     # TEST_TIERS env variable is a comma separated list of tiers to run. e.g. low, medium, high
     if ENV['TEST_TIERS']
       test_tiers = ENV['TEST_TIERS'].split(',')
       test_tiers_allowed = ENV.fetch('TEST_TIERS_ALLOWED', 'low,medium,high').split(',')
       raise 'TEST_TIERS env variable must have at least 1 tier specified. Either low, medium or high or one of the tiers listed in TEST_TIERS_ALLOWED (comma separated).' if test_tiers.count == 0
+
       test_tiers.each do |tier|
         tier_to_add = tier.strip.downcase
         raise "#{tier_to_add} not a valid test tier." unless test_tiers_allowed.include?(tier_to_add)
+
         tiers = "--tag tier_#{tier_to_add}"
-        t.rspec_opts.push(tiers)
+        task.rspec_opts.push(tiers)
       end
     else
       puts 'TEST_TIERS env variable not defined. Defaulting to run all tests.'
     end
-    t
+    task
   end
 end
 

data/lib/puppetlabs_spec_helper/tasks/check_symlinks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathspec'
 
 module PuppetlabsSpecHelper; end
@@ -33,7 +35,7 @@ class PuppetlabsSpecHelper::Tasks::CheckSymlinks
   end
 
   def ignored?(path)
-    path = path.to_s + '/' if File.directory?(path)
+    path = "#{path}/" if File.directory?(path)
 
     !ignore_pathspec.match_paths([path], Dir.pwd).empty?
   end
@@ -42,6 +44,7 @@ class PuppetlabsSpecHelper::Tasks::CheckSymlinks
     @ignore_pathspec ||= PathSpec.new(DEFAULT_IGNORED).tap do |pathspec|
       IGNORE_LIST_FILES.each do |f|
         next unless File.file?(f) && File.readable?(f)
+
         File.open(f, 'r') { |fd| pathspec.add(fd) }
       end
     end

data/lib/puppetlabs_spec_helper/tasks/fixtures.rb

@@ -1,15 +1,19 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'open3'
 require 'json'
 
 module PuppetlabsSpecHelper; end
 module PuppetlabsSpecHelper::Tasks; end
+
 module PuppetlabsSpecHelper::Tasks::FixtureHelpers
   # This is a helper for the self-symlink entry of fixtures.yml
   def source_dir
     Dir.pwd
   end
 
+  # @return [String] - the name of current module
   def module_name
     raise ArgumentError unless File.file?('metadata.json') && File.readable?('metadata.json')
 
@@ -23,16 +27,48 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     File.basename(Dir.pwd).split('-').last
   end
 
-  # cache the repositories and return a hash object
+  def module_version(path)
+    metadata_path = File.join(path, 'metadata.json')
+    raise ArgumentError unless File.file?(metadata_path) && File.readable?(metadata_path)
+
+    metadata = JSON.parse(File.read(metadata_path))
+    metadata.fetch('version', nil) || '0.0.1'
+  rescue JSON::ParserError, ArgumentError
+    logger.warn "Failed to find module version at path #{path}"
+    '0.0.1'
+  end
+
+  # @return [Hash] - returns a hash of all the fixture repositories
+  # @example
+  # {"puppetlabs-stdlib"=>{"target"=>"https://gitlab.com/puppetlabs/puppet-stdlib.git",
+  # "ref"=>nil, "branch"=>"master", "scm"=>nil,
+  # }}
   def repositories
-    unless @repositories
-      @repositories = fixtures('repositories')
-    end
-    @repositories
+    @repositories ||= fixtures('repositories') || {}
+  end
+
+  # @return [Hash] - returns a hash of all the fixture forge modules
+  # @example
+  # {"puppetlabs-stdlib"=>{"target"=>"spec/fixtures/modules/stdlib",
+  # "ref"=>nil, "branch"=>nil, "scm"=>nil,
+  # "flags"=>"--module_repository=https://myforge.example.com/", "subdir"=>nil}}
+  def forge_modules
+    @forge_modules ||= fixtures('forge_modules') || {}
+  end
+
+  # @return [Hash] - a hash of symlinks specified in the fixtures file
+  def symlinks
+    @symlinks ||= fixtures('symlinks') || {}
   end
 
+  # @return [Hash] - returns a hash with the module name and the source directory
   def auto_symlink
-    { module_name => '#{source_dir}' }
+    { module_name => "\#{source_dir}" }
+  end
+
+  # @return [Boolean] - true if the os is a windows system
+  def windows?
+    !!File::ALT_SEPARATOR
   end
 
   def fixtures(category)
@@ -77,7 +113,6 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
 
     result = {}
     if fixtures.include?(category) && !fixtures[category].nil?
-
       defaults = { 'target' => 'spec/fixtures/modules' }
 
       # load defaults from the `.fixtures.yml` `defaults` section
@@ -94,28 +129,39 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
         end
         # there should be a warning or something if it's not a hash...
         next unless opts.instance_of?(Hash)
+
         # merge our options into the defaults to get the
         # final option list
         opts = defaults.merge(opts)
 
         next unless include_repo?(opts['puppet_version'])
 
-        real_target = eval('"' + opts['target'] + '"')
-        real_source = eval('"' + opts['repo'] + '"')
+        real_target = eval("\"#{opts['target']}\"", binding, __FILE__, __LINE__) # evaluating target reference in this context (see auto_symlink)
+        real_source = eval("\"#{opts['repo']}\"", binding, __FILE__, __LINE__) # evaluating repo reference in this context (see auto_symlink)
 
-        result[real_source] = {
+        result[real_source] = validate_fixture_hash!(
           'target' => File.join(real_target, fixture),
-          'ref' => opts['ref'],
+          'ref' => opts['ref'] || opts['tag'],
           'branch' => opts['branch'],
           'scm' => opts['scm'],
           'flags' => opts['flags'],
           'subdir' => opts['subdir'],
-        }
+        )
       end
     end
     result
   end
 
+  def validate_fixture_hash!(hash)
+    # Can only validate git based scm
+    return hash unless hash['scm'] == 'git'
+
+    # Forward slashes in the ref aren't allowed. And is probably a branch name.
+    raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead." if hash['ref'] =~ %r{/}
+
+    hash
+  end
+
   def include_repo?(version_range)
     if version_range && defined?(SemanticPuppet)
       puppet_spec = Gem::Specification.find_by_name('puppet')
@@ -149,6 +195,7 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     unless File.exist?(target)
       raise "Failed to clone #{scm} repository #{remote} into #{target}"
     end
+
     result
   end
 
@@ -196,7 +243,7 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
   end
 
   def git_remote_url(target)
-    output, status = Open3.capture2e('git', '-C', target, 'remote', 'get-url', 'origin')
+    output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
     status.success? ? output.strip : nil
   end
 
@@ -219,7 +266,7 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
               else
                 Logger::INFO
               end
-      @logger = Logger.new(STDERR)
+      @logger = Logger.new($stderr)
       @logger.level = level
     end
     @logger
@@ -229,12 +276,13 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     # The problem with the relative path is that PMT doesn't expand the path properly and so passing in a relative path here
     # becomes something like C:\somewhere\backslashes/spec/fixtures/work-dir on Windows, and then PMT barfs itself.
     # This has been reported as https://tickets.puppetlabs.com/browse/PUP-4884
-    File.expand_path((ENV['MODULE_WORKING_DIR']) ? ENV['MODULE_WORKING_DIR'] : 'spec/fixtures/work-dir')
+    File.expand_path(ENV['MODULE_WORKING_DIR'] || 'spec/fixtures/work-dir')
   end
 
   # returns the current thread count that is currently active
   # a status of false or nil means the thread completed
   # so when anything else we count that as a active thread
+  # @return [Integer] - current thread count
   def current_thread_count(items)
     active_threads = items.find_all do |_item, opts|
       if opts[:thread]
@@ -247,79 +295,47 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     active_threads.count
   end
 
-  # returns the max_thread_count
-  # because we may want to limit ssh or https connections
+  # @summary Set a limit on the amount threads used, defaults to 10
+  #   MAX_FIXTURE_THREAD_COUNT can be used to set this limit
+  # @return [Integer] - returns the max_thread_count
   def max_thread_limit
-    unless @max_thread_limit
-      # the default thread count is 10 but can be
-      # raised by using environment variable MAX_FIXTURE_THREAD_COUNT
-      @max_thread_limit = if ENV['MAX_FIXTURE_THREAD_COUNT'].to_i > 0
-                            ENV['MAX_FIXTURE_THREAD_COUNT'].to_i
-                          else
-                            10 # the default
-                          end
-    end
-    @max_thread_limit
+    @max_thread_limit ||= (ENV['MAX_FIXTURE_THREAD_COUNT'] || 10).to_i
   end
-end
-include PuppetlabsSpecHelper::Tasks::FixtureHelpers
 
-desc 'Create the fixtures directory'
-task :spec_prep do
-  # Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
-  # uses this to check for Windows
-  is_windows = !!File::ALT_SEPARATOR
-  if is_windows
-    begin
-      require 'win32/dir'
-    rescue LoadError
-      $stderr.puts 'win32-dir gem not installed, falling back to executing mklink directly'
-    end
-  end
-
-  # git has a race condition creating that directory, that would lead to aborted clone operations
-  FileUtils.mkdir_p('spec/fixtures/modules')
-
-  repositories.each do |remote, opts|
-    scm = 'git'
-    target = opts['target']
-    subdir = opts['subdir']
-    ref = opts['ref']
-    scm = opts['scm'] if opts['scm']
-    branch = opts['branch'] if opts['branch']
-    flags = opts['flags']
-    # get the current active threads that are alive
-    count = current_thread_count(repositories)
-    if count < max_thread_limit
-      logger.debug "New Thread started for #{remote}"
-      # start up a new thread and store it in the opts hash
-      opts[:thread] = Thread.new do
-        if valid_repo?(scm, target, remote)
-          update_repo(scm, target)
-        else
-          clone_repo(scm, remote, target, subdir, ref, branch, flags)
+  # @param items [Hash] - a hash of either repositories or forge modules
+  # @param [Block] - the method you wish to use to download the item
+  def download_items(items)
+    items.each do |remote, opts|
+      # get the current active threads that are alive
+      count = current_thread_count(items)
+      if count < max_thread_limit
+        logger.debug "New Thread started for #{remote}"
+        # start up a new thread and store it in the opts hash
+        opts[:thread] = Thread.new do
+          yield(remote, opts)
         end
-        revision(scm, target, ref) if ref
-        remove_subdirectory(target, subdir) if subdir
+      else
+        # the last thread started should be the longest wait
+        item, item_opts = items.find_all { |_i, o| o.key?(:thread) }.last
+        logger.debug "Waiting on #{item}"
+        item_opts[:thread].join # wait for the thread to finish
+        # now that we waited lets try again
+        redo
       end
-    else
-      # the last thread started should be the longest wait
-      item, item_opts = repositories.find_all { |_i, o| o.key?(:thread) }.last
-      logger.debug "Waiting on #{item}"
-      item_opts[:thread].join # wait for the thread to finish
-      # now that we waited lets try again
-      redo
     end
+    # wait for all the threads to finish
+    items.each { |_remote, opts| opts[:thread].join }
   end
 
-  # wait for all the threads to finish
-  repositories.each { |_remote, opts| opts[:thread].join }
-
-  fixtures('symlinks').each do |target, link|
+  # @param target [String] - the target directory
+  # @param link [String] - the name of the link you wish to create
+  # works on windows and linux
+  def setup_symlink(target, link)
     link = link['target']
-    next if File.symlink?(link)
+    return if File.symlink?(link)
+
     logger.info("Creating symlink from #{link} to #{target}")
-    if is_windows
+    if windows?
       target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
       if Dir.respond_to?(:create_junction)
         Dir.create_junction(link, target)
@@ -331,7 +347,35 @@ task :spec_prep do
     end
   end
 
-  fixtures('forge_modules').each do |remote, opts|
+  # @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
+  # @param [String] - the remote url or namespace/name of the module to download
+  # @param [Hash] - list of options such as version, branch, ref
+  def download_repository(remote, opts)
+    scm = 'git'
+    target = opts['target']
+    subdir = opts['subdir']
+    ref = opts['ref']
+    scm = opts['scm'] if opts['scm']
+    branch = opts['branch'] if opts['branch']
+    flags = opts['flags']
+    if valid_repo?(scm, target, remote)
+      update_repo(scm, target)
+    else
+      clone_repo(scm, remote, target, subdir, ref, branch, flags)
+    end
+    revision(scm, target, ref) if ref
+    remove_subdirectory(target, subdir) if subdir
+  end
+
+  # @return [String] - the spec/fixtures/modules directory in the module root folder
+  def module_target_dir
+    @module_target_dir ||= File.expand_path('spec/fixtures/modules')
+  end
+
+  # @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
+  # @param [String] - the remote url or namespace/name of the module to download
+  # @param [Hash] - list of options such as version
+  def download_module(remote, opts)
     ref = ''
     flags = ''
     if opts.instance_of?(String)
@@ -342,34 +386,60 @@ task :spec_prep do
       flags = " #{opts['flags']}" if opts['flags']
     end
 
-    next if File.directory?(target)
+    return false if File.directory?(target) && (ref.empty? || opts['ref'] == module_version(target))
+
+    # The PMT cannot handle multi threaded runs due to cache directory collisons
+    # so we randomize the directory instead.
+    # Does working_dir even need to be passed?
+    Dir.mktmpdir do |working_dir|
+      command = "puppet module install#{ref}#{flags} --ignore-dependencies" \
+      ' --force' \
+      " --module_working_dir \"#{working_dir}\"" \
+      " --target-dir \"#{module_target_dir}\" \"#{remote}\""
 
-    working_dir = module_working_directory
-    target_dir = File.expand_path('spec/fixtures/modules')
+      unless system(command)
+        raise "Failed to install module #{remote} to #{module_target_dir}"
+      end
+    end
+    $CHILD_STATUS.success?
+  end
+end
 
-    command = 'puppet module install' + ref + flags + \
-              ' --ignore-dependencies' \
-              ' --force' \
-              " --module_working_dir \"#{working_dir}\"" \
-              " --target-dir \"#{target_dir}\" \"#{remote}\""
+include PuppetlabsSpecHelper::Tasks::FixtureHelpers # DSL include # rubocop:disable Style/MixinUsage
 
-    unless system(command)
-      raise "Failed to install module #{remote} to #{target_dir}"
+desc 'Create the fixtures directory'
+task :spec_prep do
+  # Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
+  # uses this to check for Windows
+  if windows?
+    begin
+      require 'win32/dir'
+    rescue LoadError
+      warn 'win32-dir gem not installed, falling back to executing mklink directly'
     end
   end
 
+  # git has a race condition creating that directory, that would lead to aborted clone operations
+  FileUtils.mkdir_p('spec/fixtures/modules')
+
+  symlinks.each { |target, link| setup_symlink(target, link) }
+
+  download_items(repositories) { |remote, opts| download_repository(remote, opts) }
+
+  download_items(forge_modules) { |remote, opts| download_module(remote, opts) }
+
   FileUtils.mkdir_p('spec/fixtures/manifests')
   FileUtils.touch('spec/fixtures/manifests/site.pp')
 end
 
 desc 'Clean up the fixtures directory'
 task :spec_clean do
-  fixtures('repositories').each do |_remote, opts|
+  repositories.each do |_remote, opts|
     target = opts['target']
     FileUtils.rm_rf(target)
   end
 
-  fixtures('forge_modules').each do |_remote, opts|
+  forge_modules.each do |_remote, opts|
     target = opts['target']
     FileUtils.rm_rf(target)
   end