puppetlabs-onceover 5.0.3

data/lib/puppetlabs-onceover/deploy.rb

@@ -0,0 +1,171 @@
+# handle local deployments (run r10k in .onceover dir)
+class PuppetlabsOnceover
+  class Deploy
+    def deploy_local(repo = PuppetlabsOnceover::Controlrepo.new, opts = {})
+      require 'puppetlabs-onceover/controlrepo'
+      require 'pathname'
+      require 'fileutils'
+      require 'multi_json'
+
+      logger.debug 'Deploying locally (R10K)...'
+
+      # Only default to running r10k if there is a Puppetfile
+      skip_r10k_default = !(File.file?(repo.puppetfile))
+      skip_r10k = opts[:skip_r10k] || skip_r10k_default
+      force     = opts[:force] || false
+      # Only attempt to resolve vendored modules if configured to do so
+      auto_vendored = opts[:auto_vendored] || false
+
+      require 'puppetlabs-onceover/vendored_modules' if auto_vendored
+
+      if repo.tempdir == nil
+        repo.tempdir = Dir.mktmpdir('r10k')
+      else
+        logger.debug "Creating #{repo.tempdir}"
+        FileUtils.mkdir_p(repo.tempdir)
+      end
+
+      # Overall plan: Copy everything in control repo to a tempdir then move the tempdir to the
+      # destination, just in case we get a recursive copy
+
+      # We need to exclude some files:
+      # * our own cache
+      # * stuff from git
+      # * stale puppet modules
+      # * random ruby gems (bundler/rvm?)
+      # * ...etc
+      excluded_dirs = [
+        File.join(repo.root, ".onceover"),
+        File.join(repo.root, ".git"),
+        File.join(repo.root, ".modules"),
+        File.join(repo.root, "vendor"),
+      ]
+      excluded_dirs << ENV['GEM_HOME'] if ENV['GEM_HOME']
+
+      # A Local modules directory likely means that the user installed r10k folders into their local control repo
+      # This conflicts with the step where onceover installs r10k after copying the control repo to the temporary
+      # .onceover directory.  The following skips copying the modules folder, to not later cause an error.
+      if File.directory?("modules")
+        logger.warn "Found modules directory in your controlrepo, skipping the copy of this directory.  If you installed modules locally using r10k, this warning is normal, if you have created modules in a local modules directory, onceover does not support testing these files, please rename this directory to conform with Puppet best practices, as this folder will conflict with Puppet's native installation of modules."
+      end
+
+      logger.debug "Creating temp dir as a staging directory for copying the controlrepo to #{repo.tempdir}"
+      temp_controlrepo = Dir.mktmpdir('controlrepo')
+
+      # onceover stores a big list of all the relative paths it places within its cache directory
+      onceover_manifest = []
+
+      Find.find repo.root do |source|
+        # work out a relative path to this source, eg:
+        # /home/geoff/control-repo/foo/bar -> foo/bar
+        relative_source = Pathname.new(source).relative_path_from(Pathname.new(repo.root)).to_s
+
+        target = File.join(temp_controlrepo, relative_source)
+
+        # ignore the path "." which represents the root of our control repo
+        if relative_source != "."
+          # add to list of files copied to cache by onceover
+          onceover_manifest << relative_source
+
+          if File.symlink?(source)
+            # Handle symlinks
+            link_target = File.readlink(source) # Get the target of the symlink
+            FileUtils.ln_s link_target, target, force: true # Create symlink at target
+          elsif File.directory? source
+            Find.prune if excluded_dirs.include? source
+            FileUtils.mkdir target
+          else
+            FileUtils.copy source, target
+          end
+        end
+      end
+      logger.debug "Writing manifest of copied controlrepo files"
+      File.write("#{temp_controlrepo}/.onceover_manifest.json", onceover_manifest.to_json)
+
+      # When using puppetfile vs deploy with r10k, we want to respect the :control_branch
+      # located in the Puppetfile. To accomplish that, we use git and find the current
+      # branch name, then replace strings within the staged puppetfile, prior to copying.
+      logger.debug "Checking current working branch"
+      git_branch = `git rev-parse --abbrev-ref HEAD`.chomp
+
+      logger.debug "found #{git_branch} as current working branch"
+      # Only try to modify Puppetfile if it exists
+      unless skip_r10k
+        FileUtils.copy repo.puppetfile, "#{temp_controlrepo}/Puppetfile"
+        puppetfile_contents = File.read("#{temp_controlrepo}/Puppetfile")
+
+        # Avoid touching thing if we don't need to
+        if /:control_branch/.match(puppetfile_contents)
+          logger.debug "replacing :control_branch mentions in the Puppetfile with #{git_branch}"
+          new_puppetfile_contents = puppetfile_contents.gsub(":control_branch", "'#{git_branch}'")
+          File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
+        end
+
+        if auto_vendored
+          tmp_puppetfile = File.join(temp_controlrepo, 'Puppetfile')
+          tmp_puppetfile_contents = File.read(tmp_puppetfile)
+          vm = PuppetlabsOnceover::VendoredModules.new({ repo: repo })
+          puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: temp_controlrepo)
+          vm.puppetfile_missing_vendored(puppetfile)
+          unless vm.missing_vendored.empty?
+            missing_slugs = vm.missing_vendored.map do |missing_mod|
+              missing_mod.keys[0]
+            end
+            logger.debug "Adding #{missing_slugs} to #{tmp_puppetfile}"
+            modlines = vm.missing_vendored.map do |missing_mod|
+              mod_slug = missing_mod.keys[0]
+              "mod '#{mod_slug}',\n  git: '#{missing_mod[mod_slug][:git]}',\n  ref: '#{missing_mod[mod_slug][:ref]}'"
+            end.join("\n")
+            File.write(tmp_puppetfile, "#{tmp_puppetfile_contents}\n# PuppetlabsOnceover Managed Vendored Modules\n#{modlines}")
+          end
+        end
+      end
+
+      # Remove all files written by the last onceover run, but not the ones
+      # added by r10k, because that's what we are trying to cache but we don't
+      # know what they are
+      old_manifest_path = "#{repo.tempdir}/#{repo.environmentpath}/production/.onceover_manifest.json"
+      if File.exist? old_manifest_path
+        logger.debug "Found manifest from previous run, parsing..."
+        old_manifest = MultiJson.load(File.read(old_manifest_path))
+        logger.debug "Removing #{old_manifest.count} files"
+        old_manifest.reverse.each do |file|
+          FileUtils.rm_f(File.join("#{repo.tempdir}/#{repo.environmentpath}/production/",file))
+        end
+      end
+      FileUtils.mkdir_p("#{repo.tempdir}/#{repo.environmentpath}")
+
+      logger.debug "Copying #{temp_controlrepo} to #{repo.tempdir}/#{repo.environmentpath}/production"
+      FileUtils.cp_r("#{temp_controlrepo}/.", "#{repo.tempdir}/#{repo.environmentpath}/production")
+      FileUtils.rm_rf(temp_controlrepo)
+
+      # Pull the trigger! If it's not already been pulled
+      if repo.tempdir and not skip_r10k
+        if File.directory?(repo.tempdir)
+          # TODO: Change this to call out to r10k directly to do this
+          # Probably something like:
+          # R10K::Settings.global_settings.evaluate(with_overrides)
+          # R10K::Action::Deploy::Environment
+          prod_dir = "#{repo.tempdir}/#{repo.environmentpath}/production"
+          Dir.chdir(prod_dir) do
+            install_cmd = []
+            install_cmd << 'r10k puppetfile install --color'
+            install_cmd << "--force" if force
+            install_cmd << "--config #{repo.r10k_config_file}" if repo.r10k_config_file
+            install_cmd << (logger.level > 0 ? "--verbose" : "--verbose debug") # Enable debugging if we're debugging
+            install_cmd << "--trace" if opts[:trace]
+            install_cmd = install_cmd.join(' ')
+            logger.debug "Running #{install_cmd} from #{prod_dir}"
+            system(install_cmd)
+            raise 'r10k could not install all required modules' unless $?.success?
+          end
+        else
+          raise "#{repo.tempdir} is not a directory"
+        end
+      end
+
+      # Return repo.tempdir for use
+      repo.tempdir
+    end
+  end
+end

data/lib/puppetlabs-onceover/group.rb

@@ -0,0 +1,78 @@
+require 'puppetlabs-onceover/class'
+require 'puppetlabs-onceover/node'
+
+class PuppetlabsOnceover
+  class Group
+    @@all = []
+
+    # Work out how to do class variables so that I can keep track of all the groups easily
+    attr_accessor :name
+    attr_accessor :members
+
+    # You need to pass in an array of strings for members, not objects, it will find the objects
+    # by itself, and yes it will reference them, not just create additional ones, woo!
+
+    def initialize(name = nil, members = [])
+      @name    = name
+      @members = []
+
+      case
+      when PuppetlabsOnceover::Group.valid_members?(members)
+        # If it's already a valid list just chuck it in there
+        @members = members
+      when members.is_a?(Hash)
+        # if it's a hash then do subtractive stuff
+        @members = PuppetlabsOnceover::TestConfig.subtractive_to_list(members)
+      when members.nil?
+        # Support empty groups yo
+        @members = []
+      else
+        # Turn it into a full list
+        # This should also handle lists that include groups
+        member_objects = members.map { |member| PuppetlabsOnceover::TestConfig.find_list(member) }
+        member_objects.flatten!
+
+        # Check that they are all the same type
+        unless PuppetlabsOnceover::Group.valid_members?(member_objects)
+          raise 'Groups must contain either all nodes or all classes. Either there was a mix, or something was spelled wrong'
+        end
+
+        # Smash it into the instance variable
+        @members = member_objects
+      end
+
+      # Finally add it to the list of all grops
+      @@all << self
+    end
+
+    def self.find(group_name)
+      @@all.each do |group|
+        if group.name == group_name
+          return group
+        end
+      end
+      nil
+    end
+
+    def self.all
+      @@all
+    end
+
+    # rubocop:disable-next Lint/DuplicateBranch
+    def self.valid_members?(members)
+      # Check that they are all the same type
+      # Also catch any errors to assume it's invalid
+      begin
+        if members.all? { |item| item.is_a?(PuppetlabsOnceover::Class) }
+          return true
+        elsif members.all? { |item| item.is_a?(PuppetlabsOnceover::Node) }
+          return true
+        else
+          return false
+        end
+      rescue StandardError
+        return false
+      end
+    end
+  end
+end

data/lib/puppetlabs-onceover/logger.rb

@@ -0,0 +1,32 @@
+require 'logging'
+
+module PuppetlabsOnceover::Logger
+  def logger
+    unless $logger
+      # here we setup a color scheme called 'bright'
+      Logging.color_scheme(
+        'bright',
+        :levels => {
+          :debug => :cyan,
+          :info => :green,
+          :warn => :yellow,
+          :error => :red,
+          :fatal => [:white, :on_red]
+        }
+      )
+
+      Logging.appenders.stdout(
+        'stdout',
+        :layout => Logging.layouts.pattern(
+          :pattern => '%l\t -> %m\n',
+          :color_scheme => 'bright'
+        )
+      )
+
+      $logger = Logging.logger['Colors']
+      $logger.add_appenders 'stdout'
+      $logger.level = :info
+    end
+    $logger
+  end
+end

data/lib/puppetlabs-onceover/node.rb

@@ -0,0 +1,72 @@
+require 'puppetlabs-onceover/controlrepo'
+
+class PuppetlabsOnceover
+  class Node
+    @@all = []
+
+    attr_accessor :name
+    attr_accessor :beaker_node
+    attr_accessor :fact_set
+    attr_accessor :trusted_set
+    attr_accessor :trusted_external_set
+
+    def initialize(name)
+      @name = name
+      @beaker_node = nil
+
+      # If we can't find the factset it will fail, so just catch that error and ignore it
+      begin
+        facts_file_index = PuppetlabsOnceover::Controlrepo.facts_files.index {|facts_file|
+          File.basename(facts_file, '.json') == name
+        }  
+        @fact_set = PuppetlabsOnceover::Node.clean_facts(PuppetlabsOnceover::Controlrepo.facts[facts_file_index])
+
+        # First see if we can find a 'trusted' hash at the top level of our factset 
+        @trusted_set = PuppetlabsOnceover::Controlrepo.trusted_facts[facts_file_index]
+        # If we don't find it, attempt to find a 'trusted.extensions' hash nested in our fact_set
+        @trusted_set = @fact_set.dig('trusted', 'extensions') if @trusted_set.nil?
+        # If we still can't find any, return an empty hash so the following doesn't blow up user written tests:
+        #   let(:trusted_facts) { trusted_facts }
+        @trusted_set = {} if @trusted_set.nil?
+
+        # First see if we can find a 'trusted_external' hash at the top level of our factset 
+        @trusted_external_set = PuppetlabsOnceover::Controlrepo.trusted_external_facts[facts_file_index]
+        # If we don't find it, attempt to find a 'trusted.external' hash nested in our fact_set
+        @trusted_external_set = @fact_set.dig('trusted', 'external') if @trusted_external_set.nil?
+        # If we still can't find any, return an empty hash so the following doesn't blow up user written tests:
+        #   let(:trusted_external_data) { trusted_external_data }
+        @trusted_external_set = {} if @trusted_external_set.nil?
+      rescue TypeError
+        @fact_set = {}
+        @trusted_set = {}
+        @trusted_external_set = {}
+      end
+
+      @@all << self
+    end
+
+    def self.find(node_name)
+      @@all.each do |node|
+        if node_name.is_a?(PuppetlabsOnceover::Node)
+          if node = node_name
+            return node
+          end
+        elsif node.name == node_name
+          return node
+        end
+      end
+      logger.warn "Node #{node_name} not found"
+      nil
+    end
+
+    def self.all
+      @@all
+    end
+
+    # This method ensures that all facts are valid and clean anoything that we can't handle
+    def self.clean_facts(factset)
+      factset.delete('environment')
+      factset
+    end
+  end
+end

data/lib/puppetlabs-onceover/rake_tasks.rb

@@ -0,0 +1,95 @@
+require 'puppetlabs-onceover/controlrepo'
+require 'pathname'
+
+@repo   = nil
+@config = nil
+
+desc 'Writes a `fixtures.yml` file based on the Puppetfile'
+task :generate_fixtures do
+  repo = PuppetlabsOnceover::Controlrepo.new
+  if File.exist?(File.expand_path('./.fixtures.yml', repo.root))
+    raise ".fixtures.yml already exits, we won't overwrite because we are scared"
+  end
+
+  File.write(File.expand_path('./.fixtures.yml', repo.root), repo.fixtures)
+end
+
+desc "Modifies your `hiera.yaml` to point at the hieradata relative to its position."
+task :hiera_setup do
+  repo = PuppetlabsOnceover::Controlrepo.new
+  current_config = repo.hiera_config
+  current_config.each do |key, value|
+    if value.is_a?(Hash)
+      if value.has_key?(:datadir)
+        hiera_config_path = Pathname.new(File.expand_path('..', repo.hiera_config_file))
+        current_config[key][:datadir] = Pathname.new(repo.hiera_data).relative_path_from(hiera_config_path).to_s
+      end
+    end
+  end
+  puts "Changing hiera config from \n#{repo.hiera_config}\nto\n#{current_config}"
+  repo.hiera_config = current_config
+end
+
+task :controlrepo_details do
+  require 'puppetlabs-onceover/controlrepo'
+  puts PuppetlabsOnceover::Controlrepo.new.to_s
+end
+
+task :generate_onceover_yaml do
+  require 'puppetlabs-onceover/controlrepo'
+  repo = PuppetlabsOnceover::Controlrepo.new
+  template_dir = File.expand_path('../../templates', File.dirname(__FILE__))
+  onceover_yaml_template = File.read(File.expand_path('./controlrepo.yaml.erb', template_dir))
+  puts ERB.new(onceover_yaml_template, nil, '-').result(binding)
+end
+
+task :generate_nodesets do
+  warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
+
+  require 'puppetlabs-onceover/beaker'
+  require 'net/http'
+  require 'multi_json'
+
+  repo = PuppetlabsOnceover::Controlrepo.new
+
+  puts "HOSTS:"
+
+  repo.facts.each do |fact_set|
+    node_name = File.basename(repo.facts_files[repo.facts.index(fact_set)], '.json')
+    boxname   = PuppetlabsOnceover::Beaker.facts_to_vagrant_box(fact_set)
+    platform  = PuppetlabsOnceover::Beaker.facts_to_platform(fact_set)
+
+    uri = URI("https://atlas.hashicorp.com:443/api/v1/box/#{boxname}")
+    request = Net::HTTP.new(uri.host, uri.port)
+    request.use_ssl = true
+    response = request.get(uri)
+
+    url = 'URL goes here'
+
+    if response.code == "404"
+      comment_out = true
+    else
+      comment_out = false
+      box_info = MultiJson.load(response.body)
+      box_info['current_version']['providers'].each do |provider|
+        if provider['name'] == 'virtualbox'
+          url = provider['original_url']
+        end
+      end
+    end
+
+    # Use an ERB template
+    template_dir = File.expand_path('../../templates', File.dirname(__FILE__))
+    fixtures_template = File.read(File.expand_path('./nodeset.yaml.erb', template_dir))
+    puts ERB.new(fixtures_template, nil, '-').result(binding)
+  end
+end
+
+desc 'Create cache for vendored modules'
+task :generate_vendor_cache do
+  require 'puppetlabs-onceover/controlrepo'
+  require 'puppetlabs-onceover/vendored_modules'
+
+  repo = PuppetlabsOnceover::Controlrepo.new(debug: true)
+  PuppetlabsOnceover::VendoredModules.new({ repo: repo, cachedir: File.join(repo.spec_dir, 'vendored_modules'), force_update: true })
+end