puppetlabs_spec_helper 2.7.0 → 3.0.0

data/lib/puppetlabs_spec_helper/tasks/fixtures.rb

@@ -1,58 +1,109 @@
+# frozen_string_literal: true
+
 require 'yaml'
+require 'open3'
+require 'json'
 
 module PuppetlabsSpecHelper; end
 module PuppetlabsSpecHelper::Tasks; end
+
 module PuppetlabsSpecHelper::Tasks::FixtureHelpers
   # This is a helper for the self-symlink entry of fixtures.yml
   def source_dir
     Dir.pwd
   end
 
-  # cache the repositories and return a hash object
+  # @return [String] - the name of current module
+  def module_name
+    raise ArgumentError unless File.file?('metadata.json') && File.readable?('metadata.json')
+
+    metadata = JSON.parse(File.read('metadata.json'))
+    metadata_name = metadata.fetch('name', nil) || ''
+
+    raise ArgumentError if metadata_name.empty?
+
+    metadata_name.split('-').last
+  rescue JSON::ParserError, ArgumentError
+    File.basename(Dir.pwd).split('-').last
+  end
+
+  def module_version(path)
+    metadata_path = File.join(path, 'metadata.json')
+    raise ArgumentError unless File.file?(metadata_path) && File.readable?(metadata_path)
+
+    metadata = JSON.parse(File.read(metadata_path))
+    metadata.fetch('version', nil) || '0.0.1'
+  rescue JSON::ParserError, ArgumentError
+    logger.warn "Failed to find module version at path #{path}"
+    '0.0.1'
+  end
+
+  # @return [Hash] - returns a hash of all the fixture repositories
+  # @example
+  # {"puppetlabs-stdlib"=>{"target"=>"https://gitlab.com/puppetlabs/puppet-stdlib.git",
+  # "ref"=>nil, "branch"=>"master", "scm"=>nil,
+  # }}
   def repositories
-    unless @repositories
-      @repositories = fixtures('repositories')
-    end
-    @repositories
+    @repositories ||= fixtures('repositories') || {}
+  end
+
+  # @return [Hash] - returns a hash of all the fixture forge modules
+  # @example
+  # {"puppetlabs-stdlib"=>{"target"=>"spec/fixtures/modules/stdlib",
+  # "ref"=>nil, "branch"=>nil, "scm"=>nil,
+  # "flags"=>"--module_repository=https://myforge.example.com/", "subdir"=>nil}}
+  def forge_modules
+    @forge_modules ||= fixtures('forge_modules') || {}
+  end
+
+  # @return [Hash] - a hash of symlinks specified in the fixtures file
+  def symlinks
+    @symlinks ||= fixtures('symlinks') || {}
   end
 
+  # @return [Hash] - returns a hash with the module name and the source directory
   def auto_symlink
-    { File.basename(Dir.pwd).split('-').last => '#{source_dir}' }
+    { module_name => "\#{source_dir}" }
+  end
+
+  # @return [Boolean] - true if the os is a windows system
+  def windows?
+    !!File::ALT_SEPARATOR
   end
 
   def fixtures(category)
-    if ENV['FIXTURES_YML']
-      fixtures_yaml = ENV['FIXTURES_YML']
-    elsif File.exists?('.fixtures.yml')
-      fixtures_yaml = '.fixtures.yml'
-    elsif File.exists?('.fixtures.yaml')
-      fixtures_yaml = '.fixtures.yaml'
-    else
-      fixtures_yaml = false
-    end
+    fixtures_yaml = if ENV['FIXTURES_YML']
+                      ENV['FIXTURES_YML']
+                    elsif File.exist?('.fixtures.yml')
+                      '.fixtures.yml'
+                    elsif File.exist?('.fixtures.yaml')
+                      '.fixtures.yaml'
+                    else
+                      false
+                    end
 
     begin
-      if fixtures_yaml
-        fixtures = YAML.load_file(fixtures_yaml) || { 'fixtures' => {} }
-      else
-        fixtures = { 'fixtures' => {} }
-      end
+      fixtures = if fixtures_yaml
+                   YAML.load_file(fixtures_yaml) || { 'fixtures' => {} }
+                 else
+                   { 'fixtures' => {} }
+                 end
     rescue Errno::ENOENT
-      fail("Fixtures file not found: '#{fixtures_yaml}'")
+      raise("Fixtures file not found: '#{fixtures_yaml}'")
     rescue Psych::SyntaxError => e
-      fail("Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}")
+      raise("Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}")
     end
 
     unless fixtures.include?('fixtures')
       # File is non-empty, but does not specify fixtures
-      fail("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
+      raise("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
     end
 
-    if fixtures.include? 'defaults'
-      fixture_defaults = fixtures['defaults']
-    else
-      fixture_defaults = {}
-    end
+    fixture_defaults = if fixtures.include? 'defaults'
+                         fixtures['defaults']
+                       else
+                         {}
+                       end
 
     fixtures = fixtures['fixtures']
 
@@ -61,9 +112,8 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     end
 
     result = {}
-    if fixtures.include? category and fixtures[category] != nil
-
-      defaults = { "target" => "spec/fixtures/modules" }
+    if fixtures.include?(category) && !fixtures[category].nil?
+      defaults = { 'target' => 'spec/fixtures/modules' }
 
       # load defaults from the `.fixtures.yml` `defaults` section
       # for the requested category and merge them into my defaults
@@ -75,25 +125,56 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
         # convert a simple string fixture to a hash, by
         # using the string fixture as the `repo` option of the hash.
         if opts.instance_of?(String)
-          opts = { "repo" => opts }
+          opts = { 'repo' => opts }
         end
         # there should be a warning or something if it's not a hash...
-        if opts.instance_of?(Hash)
-          # merge our options into the defaults to get the
-          # final option list
-          opts = defaults.merge(opts)
+        next unless opts.instance_of?(Hash)
 
-          real_target = eval('"'+opts["target"]+'"')
-          real_source = eval('"'+opts["repo"]+'"')
+        # merge our options into the defaults to get the
+        # final option list
+        opts = defaults.merge(opts)
 
-          result[real_source] = { "target" => File.join(real_target,fixture), "ref" => opts["ref"], "branch" => opts["branch"], "scm" => opts["scm"], "flags" => opts["flags"], "subdir" => opts["subdir"]}
-        end
+        next unless include_repo?(opts['puppet_version'])
+
+        real_target = eval("\"#{opts['target']}\"", binding, __FILE__, __LINE__) # evaluating target reference in this context (see auto_symlink)
+        real_source = eval("\"#{opts['repo']}\"", binding, __FILE__, __LINE__) # evaluating repo reference in this context (see auto_symlink)
+
+        result[real_source] = validate_fixture_hash!(
+          'target' => File.join(real_target, fixture),
+          'ref' => opts['ref'] || opts['tag'],
+          'branch' => opts['branch'],
+          'scm' => opts['scm'],
+          'flags' => opts['flags'],
+          'subdir' => opts['subdir'],
+        )
       end
     end
-    return result
+    result
+  end
+
+  def validate_fixture_hash!(hash)
+    # Can only validate git based scm
+    return hash unless hash['scm'] == 'git'
+
+    # Forward slashes in the ref aren't allowed. And is probably a branch name.
+    raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead." if hash['ref'] =~ %r{/}
+
+    hash
+  end
+
+  def include_repo?(version_range)
+    if version_range && defined?(SemanticPuppet)
+      puppet_spec = Gem::Specification.find_by_name('puppet')
+      puppet_version = SemanticPuppet::Version.parse(puppet_spec.version.to_s)
+
+      constraint = SemanticPuppet::VersionRange.parse(version_range)
+      constraint.include?(puppet_version)
+    else
+      true
+    end
   end
 
-  def clone_repo(scm, remote, target, subdir=nil, ref=nil, branch=nil, flags = nil)
+  def clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil)
     args = []
     case scm
     when 'hg'
@@ -108,15 +189,34 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
       args.push(flags) if flags
       args.push(remote, target)
     else
-      fail "Unfortunately #{scm} is not supported yet"
+      raise "Unfortunately #{scm} is not supported yet"
     end
     result = system("#{scm} #{args.flatten.join ' '}")
-    unless File::exists?(target)
-      fail "Failed to clone #{scm} repository #{remote} into #{target}"
+    unless File.exist?(target)
+      raise "Failed to clone #{scm} repository #{remote} into #{target}"
     end
+
     result
   end
 
+  def update_repo(scm, target)
+    args = case scm
+           when 'hg'
+             ['pull']
+           when 'git'
+             ['fetch'].tap do |git_args|
+               git_args << '--unshallow' if shallow_git_repo?
+             end
+           else
+             raise "Unfortunately #{scm} is not supported yet"
+           end
+    system("#{scm} #{args.flatten.join(' ')}", chdir: target)
+  end
+
+  def shallow_git_repo?
+    File.file?(File.join('.git', 'shallow'))
+  end
+
   def revision(scm, target, ref)
     args = []
     case scm
@@ -125,18 +225,35 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     when 'git'
       args.push('reset', '--hard', ref)
     else
-      fail "Unfortunately #{scm} is not supported yet"
+      raise "Unfortunately #{scm} is not supported yet"
     end
-    system("cd #{target} && #{scm} #{args.flatten.join ' '}")
+    result = system("#{scm} #{args.flatten.join ' '}", chdir: target)
+    raise "Invalid ref #{ref} for #{target}" unless result
+  end
+
+  def valid_repo?(scm, target, remote)
+    return false unless File.directory?(target)
+    return true if scm == 'hg'
+
+    return true if git_remote_url(target) == remote
+
+    warn "Git remote for #{target} has changed, recloning repository"
+    FileUtils.rm_rf(target)
+    false
+  end
+
+  def git_remote_url(target)
+    output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
+    status.success? ? output.strip : nil
   end
 
   def remove_subdirectory(target, subdir)
     unless subdir.nil?
-      Dir.mktmpdir {|tmpdir|
-         FileUtils.mv(Dir.glob("#{target}/#{subdir}/{.[^\.]*,*}"), tmpdir)
-         FileUtils.rm_rf("#{target}/#{subdir}")
-         FileUtils.mv(Dir.glob("#{tmpdir}/{.[^\.]*,*}"), "#{target}")
-      }
+      Dir.mktmpdir do |tmpdir|
+        FileUtils.mv(Dir.glob("#{target}/#{subdir}/{.[^\.]*,*}"), tmpdir)
+        FileUtils.rm_rf("#{target}/#{subdir}")
+        FileUtils.mv(Dir.glob("#{tmpdir}/{.[^\.]*,*}"), target.to_s)
+      end
     end
   end
 
@@ -144,12 +261,12 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
   def logger
     unless @logger
       require 'logger'
-      if ENV['ENABLE_LOGGER']
-         level = Logger::DEBUG
-       else
-         level = Logger::INFO
-      end
-      @logger = Logger.new(STDERR)
+      level = if ENV['ENABLE_LOGGER']
+                Logger::DEBUG
+              else
+                Logger::INFO
+              end
+      @logger = Logger.new($stderr)
       @logger.level = level
     end
     @logger
@@ -159,14 +276,15 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     # The problem with the relative path is that PMT doesn't expand the path properly and so passing in a relative path here
     # becomes something like C:\somewhere\backslashes/spec/fixtures/work-dir on Windows, and then PMT barfs itself.
     # This has been reported as https://tickets.puppetlabs.com/browse/PUP-4884
-    File.expand_path(ENV['MODULE_WORKING_DIR'] ? ENV['MODULE_WORKING_DIR'] : 'spec/fixtures/work-dir')
+    File.expand_path(ENV['MODULE_WORKING_DIR'] || 'spec/fixtures/work-dir')
   end
 
   # returns the current thread count that is currently active
   # a status of false or nil means the thread completed
   # so when anything else we count that as a active thread
+  # @return [Integer] - current thread count
   def current_thread_count(items)
-    active_threads = items.find_all do |item, opts|
+    active_threads = items.find_all do |_item, opts|
       if opts[:thread]
         opts[:thread].status
       else
@@ -177,152 +295,168 @@ module PuppetlabsSpecHelper::Tasks::FixtureHelpers
     active_threads.count
   end
 
-  # returns the max_thread_count
-  # because we may want to limit ssh or https connections
+  # @summary Set a limit on the amount threads used, defaults to 10
+  #   MAX_FIXTURE_THREAD_COUNT can be used to set this limit
+  # @return [Integer] - returns the max_thread_count
   def max_thread_limit
-    unless @max_thread_limit
-      # the default thread count is 10 but can be
-      # raised by using environment variable MAX_FIXTURE_THREAD_COUNT
-      if ENV['MAX_FIXTURE_THREAD_COUNT'].to_i > 0
-        @max_thread_limit = ENV['MAX_FIXTURE_THREAD_COUNT'].to_i
+    @max_thread_limit ||= (ENV['MAX_FIXTURE_THREAD_COUNT'] || 10).to_i
+  end
+
+  # @param items [Hash] - a hash of either repositories or forge modules
+  # @param [Block] - the method you wish to use to download the item
+  def download_items(items)
+    items.each do |remote, opts|
+      # get the current active threads that are alive
+      count = current_thread_count(items)
+      if count < max_thread_limit
+        logger.debug "New Thread started for #{remote}"
+        # start up a new thread and store it in the opts hash
+        opts[:thread] = Thread.new do
+          yield(remote, opts)
+        end
       else
-        @max_thread_limit = 10 # the default
+        # the last thread started should be the longest wait
+        item, item_opts = items.find_all { |_i, o| o.key?(:thread) }.last
+        logger.debug "Waiting on #{item}"
+        item_opts[:thread].join # wait for the thread to finish
+        # now that we waited lets try again
+        redo
       end
     end
-    @max_thread_limit
+    # wait for all the threads to finish
+    items.each { |_remote, opts| opts[:thread].join }
   end
 
-  def check_directory_for_symlinks(dir='.')
-    dir = Pathname.new(dir) unless dir.is_a?(Pathname)
-    results = []
+  # @param target [String] - the target directory
+  # @param link [String] - the name of the link you wish to create
+  # works on windows and linux
+  def setup_symlink(target, link)
+    link = link['target']
+    return if File.symlink?(link)
 
-    dir.each_child(true) do |child|
-      if child.symlink?
-        results << child
-      elsif child.directory? && child.basename.to_s != '.git'
-        results.concat(check_directory_for_symlinks(child))
+    logger.info("Creating symlink from #{link} to #{target}")
+    if windows?
+      target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
+      if Dir.respond_to?(:create_junction)
+        Dir.create_junction(link, target)
+      else
+        system("call mklink /J \"#{link.tr('/', '\\')}\" \"#{target.tr('/', '\\')}\"")
       end
-    end
-
-    results
-  end
-end
-include PuppetlabsSpecHelper::Tasks::FixtureHelpers
-
-desc "Create the fixtures directory"
-task :spec_prep do
-  # Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
-  # uses this to check for Windows
-  is_windows = !!File::ALT_SEPARATOR
-  if is_windows
-    begin
-      require 'win32/dir'
-    rescue LoadError
-      $stderr.puts "win32-dir gem not installed, falling back to executing mklink directly"
+    else
+      FileUtils.ln_sf(target, link)
     end
   end
 
-  # git has a race condition creating that directory, that would lead to aborted clone operations
-  FileUtils::mkdir_p("spec/fixtures/modules")
-
-  repositories.each do |remote, opts|
+  # @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
+  # @param [String] - the remote url or namespace/name of the module to download
+  # @param [Hash] - list of options such as version, branch, ref
+  def download_repository(remote, opts)
     scm = 'git'
-    target = opts["target"]
-    subdir = opts["subdir"]
-    ref = opts["ref"]
-    scm = opts["scm"] if opts["scm"]
-    branch = opts["branch"] if opts["branch"]
-    flags = opts["flags"]
-    # get the current active threads that are alive
-    count = current_thread_count(repositories)
-    if count < max_thread_limit
-      logger.debug "New Thread started for #{remote}"
-      # start up a new thread and store it in the opts hash
-      opts[:thread] = Thread.new do
-        clone_repo(scm, remote, target, subdir, ref, branch, flags)
-        revision(scm, target, ref) if ref
-        remove_subdirectory(target, subdir) if subdir
-      end
+    target = opts['target']
+    subdir = opts['subdir']
+    ref = opts['ref']
+    scm = opts['scm'] if opts['scm']
+    branch = opts['branch'] if opts['branch']
+    flags = opts['flags']
+    if valid_repo?(scm, target, remote)
+      update_repo(scm, target)
     else
-      # the last thread started should be the longest wait
-      item, item_opts = repositories.find_all {|i,o| o.has_key?(:thread)}.last
-      logger.debug "Waiting on #{item}"
-      item_opts[:thread].join  # wait for the thread to finish
-      # now that we waited lets try again
-      redo
+      clone_repo(scm, remote, target, subdir, ref, branch, flags)
     end
+    revision(scm, target, ref) if ref
+    remove_subdirectory(target, subdir) if subdir
   end
 
-  # wait for all the threads to finish
-  repositories.each {|remote, opts| opts[:thread].join }
-
-  fixtures("symlinks").each do |target, link|
-    link = link['target']
-    unless File.symlink?(link)
-      logger.info("Creating symlink from #{link} to #{target}")
-      if is_windows
-        target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
-        if Dir.respond_to?(:create_junction)
-          Dir.create_junction(link, target)
-        else
-          system("call mklink /J \"#{link.gsub('/', '\\')}\" \"#{target.gsub('/', '\\')}\"")
-        end
-      else
-        FileUtils::ln_sf(target, link)
-      end
-    end
+  # @return [String] - the spec/fixtures/modules directory in the module root folder
+  def module_target_dir
+    @module_target_dir ||= File.expand_path('spec/fixtures/modules')
   end
 
-  fixtures("forge_modules").each do |remote, opts|
-    ref = ""
-    flags = ""
+  # @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
+  # @param [String] - the remote url or namespace/name of the module to download
+  # @param [Hash] - list of options such as version
+  def download_module(remote, opts)
+    ref = ''
+    flags = ''
     if opts.instance_of?(String)
       target = opts
     elsif opts.instance_of?(Hash)
-      target = opts["target"]
-      ref = " --version #{opts['ref']}" if not opts['ref'].nil?
+      target = opts['target']
+      ref = " --version #{opts['ref']}" unless opts['ref'].nil?
       flags = " #{opts['flags']}" if opts['flags']
     end
-    next if File::exists?(target)
 
-    working_dir = module_working_directory
-    target_dir = File.expand_path('spec/fixtures/modules')
+    return false if File.directory?(target) && (ref.empty? || opts['ref'] == module_version(target))
 
-    command = "puppet module install" + ref + flags + \
-      " --ignore-dependencies" \
-      " --force" \
+    # The PMT cannot handle multi threaded runs due to cache directory collisons
+    # so we randomize the directory instead.
+    # Does working_dir even need to be passed?
+    Dir.mktmpdir do |working_dir|
+      command = "puppet module install#{ref}#{flags} --ignore-dependencies" \
+      ' --force' \
       " --module_working_dir \"#{working_dir}\"" \
-      " --target-dir \"#{target_dir}\" \"#{remote}\""
+      " --target-dir \"#{module_target_dir}\" \"#{remote}\""
 
-    unless system(command)
-      fail "Failed to install module #{remote} to #{target_dir}"
+      unless system(command)
+        raise "Failed to install module #{remote} to #{module_target_dir}"
+      end
     end
+    $CHILD_STATUS.success?
   end
+end
 
-  FileUtils::mkdir_p("spec/fixtures/manifests")
-  FileUtils::touch("spec/fixtures/manifests/site.pp")
+include PuppetlabsSpecHelper::Tasks::FixtureHelpers # DSL include # rubocop:disable Style/MixinUsage
+
+desc 'Create the fixtures directory'
+task :spec_prep do
+  # Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
+  # uses this to check for Windows
+  if windows?
+    begin
+      require 'win32/dir'
+    rescue LoadError
+      warn 'win32-dir gem not installed, falling back to executing mklink directly'
+    end
+  end
+
+  # git has a race condition creating that directory, that would lead to aborted clone operations
+  FileUtils.mkdir_p('spec/fixtures/modules')
+
+  symlinks.each { |target, link| setup_symlink(target, link) }
+
+  download_items(repositories) { |remote, opts| download_repository(remote, opts) }
+
+  download_items(forge_modules) { |remote, opts| download_module(remote, opts) }
+
+  FileUtils.mkdir_p('spec/fixtures/manifests')
+  FileUtils.touch('spec/fixtures/manifests/site.pp')
 end
 
-desc "Clean up the fixtures directory"
+desc 'Clean up the fixtures directory'
 task :spec_clean do
-  fixtures("repositories").each do |remote, opts|
-    target = opts["target"]
-    FileUtils::rm_rf(target)
+  repositories.each do |_remote, opts|
+    target = opts['target']
+    FileUtils.rm_rf(target)
   end
 
-  fixtures("forge_modules").each do |remote, opts|
-    target = opts["target"]
-    FileUtils::rm_rf(target)
+  forge_modules.each do |_remote, opts|
+    target = opts['target']
+    FileUtils.rm_rf(target)
   end
 
-  FileUtils::rm_rf(module_working_directory)
+  FileUtils.rm_rf(module_working_directory)
 
-  fixtures("symlinks").each do |source, opts|
-    target = opts["target"]
-    FileUtils::rm_f(target)
+  Rake::Task[:spec_clean_symlinks].invoke
+
+  if File.zero?('spec/fixtures/manifests/site.pp')
+    FileUtils.rm_f('spec/fixtures/manifests/site.pp')
   end
+end
 
-  if File.zero?("spec/fixtures/manifests/site.pp")
-    FileUtils::rm_f("spec/fixtures/manifests/site.pp")
+desc 'Clean up any fixture symlinks'
+task :spec_clean_symlinks do
+  fixtures('symlinks').each do |_source, opts|
+    target = opts['target']
+    FileUtils.rm_f(target)
   end
 end

data/lib/puppetlabs_spec_helper/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module PuppetlabsSpecHelper
-  VERSION = "2.7.0"
+  VERSION = '3.0.0'
 
   # compat for pre-1.2.0 users; deprecated
   module Version

data/puppet_spec_helper.rb

@@ -1,4 +1,6 @@
-$:.unshift File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
 
 require 'puppetlabs_spec_helper/puppet_spec_helper'