puppetlabs-onceover 5.0.3

data/lib/puppetlabs-onceover/rspec/formatters.rb

@@ -0,0 +1,315 @@
+require 'rspec'
+require 'rspec-puppet'
+require 'pathname'
+require 'English'
+
+class PuppetlabsOnceoverFormatter
+  RSpec::Core::Formatters.register(
+    self,
+    :example_group_started,
+    :example_passed,
+    :example_failed,
+    :example_pending,
+    :dump_failures,
+  )
+
+  COMPILATION_ERROR      = %r{error during compilation: (?<error>.*)}
+  # rubocop:disable-next Lint/MixedRegexpCaptureTypes
+  ERROR_WITH_LOCATION    = %r{(?<error>.*?)\s(at )?(\((file: (?<file>.*?), )?line: (?<line>\d+)(, column: (?<column>\d+))?\))(; )?}
+  ERROR_WITHOUT_LOCATION = %r{(?<error>.*?)\son node}
+
+  def initialize output
+    @output        = output
+    @previous_role = nil
+  end
+
+  def example_group_started notification
+    if notification.group.parent_groups == [notification.group]
+      # If this is the highest level group (The role)
+      role = notification.group.description
+      if role != @previous_role
+        @output << "\n"
+        @output << class_name("#{notification.group.description}:")
+
+        # Calculate the padding required
+        padding = (longest_group - role.length) + 1
+        # Create padding
+        padding.times { @output << ' ' }
+
+        # Save the role name
+        @previous_role = role
+      end
+    else
+      # If not then this will be a test for that role
+      @output << '? '
+    end
+  end
+
+  def example_passed notification
+    @output << "\b\b"
+    @output << "#{green('P')} "
+  end
+
+  def example_failed notification
+    @output << "\b\b"
+    @output << "#{red('F')} "
+  end
+
+  def example_pending notification
+    @output << "\b\b"
+    @output << "#{yellow('?')} "
+  end
+
+  def dump_failures notification
+    require 'puppetlabs-onceover/controlrepo'
+
+    failures = extract_failures(notification)
+
+    # Put some spacing before the results
+    @output << "\n\n\n"
+
+    failures.each do |name, errors|
+      @output << PuppetlabsOnceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding)
+    end
+
+    @output << "\n"
+  end
+
+  # rubocop:disable Style/CombinableLoops
+  #
+  # This method takes a notification and formats it into a hash that can be
+  # printed easily
+  def extract_failures notification
+    # Group by role
+    grouped = notification.failed_examples.group_by { |e| e.metadata[:example_group][:parent_example_group][:description]}
+
+    # Further group by error
+    grouped.each do |role, failures|
+      grouped[role] = failures.group_by { |f| f.metadata[:execution_result].exception.to_s }
+    end
+
+    # Extract the errors and remove all RSpec objects
+    grouped.each do |role, failures|
+      grouped[role] = failures.map { |_description, fails| extract_failure_data(fails)}.flatten
+    end
+
+    grouped
+  end
+  # rubocop:enable Style/CombinableLoops
+
+  # Extaracts data out of RSpec failres
+  def extract_failure_data(fails)
+    # The only difference between these failures should be the factsets that it
+    # failed on. Extract that list then just use the first failure for the rest
+    # of the data as it should be the same
+    metadata          = fails[0].metadata
+    raw_error         = metadata[:execution_result].exception.to_s
+    factsets          = fails.map { |f| f.metadata[:example_group][:description].gsub('using fact set ','') }
+    results           = parse_errors(raw_error)
+    # Add the details of the factsets tio each result
+    results.map do |r|
+      r[:factsets] = factsets
+      r
+    end
+  end
+
+  # Parses information out of a string error
+  def parse_errors(raw_error)
+    # Check if the error is a compilation error
+    match = COMPILATION_ERROR.match(raw_error)
+    if match
+      compilation_error = match['error']
+      # Check if we car parse it
+      if ERROR_WITH_LOCATION.match(compilation_error)
+        scanned_errors = match['error'].scan(ERROR_WITH_LOCATION)
+
+        # Delete any matches where there was no error text
+        scanned_errors.delete_if { |e| e.first.empty? }
+
+        scanned_errors.map do |error_matches|
+          {
+            text: error_matches[0],
+            file: calculate_relative_source(error_matches[1]),
+            line: error_matches[2],
+            column: error_matches[3],
+          }
+        end
+      elsif ERROR_WITHOUT_LOCATION.match(compilation_error)
+        scanned_errors = match['error'].scan(ERROR_WITHOUT_LOCATION)
+
+        # Delete any matches where there was no error text
+        scanned_errors.delete_if { |e| e.first.empty? }
+
+        scanned_errors.map do |error_matches|
+          {
+            text: error_matches[0],
+          }
+        end
+      else
+        [{
+          text: raw_error,
+        }]
+      end
+    else
+      [{
+        text: raw_error,
+      }]
+    end
+  end
+
+  # This method calculates where the original source file is relative to the
+  # user's current location. This is more compliacted than it sounds because
+  # if we are running from the root of the controlrepo and we have an error in:
+  #
+  # /Users/dylan/git/puppet_controlrepo/.onceover/etc/puppetlabs/code/environments/production/site/role/manifests/lb.pp
+  #
+  # We need that to end up pointing at the original source file not the cached
+  # one i.e.
+  #
+  # site/role/manifests/lb.pp
+  #
+  def calculate_relative_source(file)
+    return nil if file.nil?
+
+    file            = Pathname.new(file)
+    tempdir         = Pathname.new(RSpec.configuration.onceover_tempdir)
+    root            = Pathname.new(RSpec.configuration.onceover_root)
+    environmentpath = Pathname.new(RSpec.configuration.onceover_environmentpath)
+
+    # Calculate the full relative path
+    file.relative_path_from(tempdir + environmentpath + "production").to_s
+  end
+
+  # Below are defined the styles for the output
+  def class_name(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :bold)
+  end
+
+  def black(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :black)
+  end
+
+  def red(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :red)
+  end
+
+  def green(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :green)
+  end
+
+  def yellow(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :yellow)
+  end
+
+  def blue(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :blue)
+  end
+
+  def magenta(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :magenta)
+  end
+
+  def cyan(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :cyan)
+  end
+
+  def white(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :white)
+  end
+
+  def bold(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :bold)
+  end
+
+  def longest_group
+    RSpec.configuration.world.example_groups.max { |a,b| a.description.length <=> b.description.length}.description.length
+  end
+end
+
+class PuppetlabsOnceoverFormatterParallel < PuppetlabsOnceoverFormatter
+  require 'yaml'
+
+  RSpec::Core::Formatters.register(
+    self,
+    :example_group_started,
+    :example_passed,
+    :example_failed,
+    :example_pending,
+    :dump_failures,
+  )
+
+  def example_group_started notification
+    # Do nothing
+  end
+
+  def example_passed notification
+    @output << green('P')
+    @output.flush
+  end
+
+  def example_failed notification
+    @output << red('F')
+    @output.flush
+  end
+
+  def example_pending notification
+    @output << yellow('?')
+    @output.flush
+  end
+
+  def dump_failures notification
+    # Create a random string
+    require 'securerandom'
+    random_string = SecureRandom.hex
+
+    # Ensure that the folder exists
+    FileUtils.mkdir_p "#{RSpec.configuration.onceover_tempdir}/parallel"
+
+    # Dump the notification to a unique file
+    File.write("#{RSpec.configuration.onceover_tempdir}/parallel/results-#{random_string}.yaml", extract_failures(notification).to_yaml)
+  end
+
+  def output_results(directory)
+    require 'rspec/core/example'
+    # Read all yaml files
+    results = {}
+    files   = Dir["#{directory}/*.yaml"]
+
+    # Merge data
+    roles = files.reduce({}) do |errs, file|
+      # Read all files and merge them
+      errs.merge(YAML.load_file(file)) {|key, oldval, newval| [oldval, newval].flatten }
+    end
+
+    # Delete files from the disk
+    files.each { |f| File.delete(f) }
+
+    @output << "\n\n\n"
+
+    # Output errors
+    roles.each do |name, errors|
+      @output << PuppetlabsOnceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding)
+    end
+    @output << "\n"
+  end
+end
+
+class FailureCollector
+  RSpec::Core::Formatters.register self, :dump_failures
+
+  def initialize(output)
+    FileUtils.touch(File.expand_path("#{RSpec.configuration.onceover_tempdir}/failures.out"))
+  end
+
+  def dump_failures(failures)
+    File.open(File.expand_path("#{RSpec.configuration.onceover_tempdir}/failures.out"), 'a') { |f|
+      failures.failed_examples.each do |fe|
+        f.puts
+        f.puts "#{fe.metadata[:description]}"
+        f.puts "#{fe.metadata[:execution_result].exception.to_s}"
+        f.puts "#{fe.metadata[:file_path]}:#{fe.metadata[:line_number]}"
+        f.puts "------------------------------------------------------"
+      end
+    }
+  end
+end

data/lib/puppetlabs-onceover/runner.rb

@@ -0,0 +1,139 @@
+require 'backticks'
+require 'io/console'
+
+class PuppetlabsOnceover
+  class Runner
+    attr_reader :repo
+    attr_reader :config
+
+    def initialize(repo, config, mode = [:spec, :acceptance])
+      @repo   = repo
+      @config = config
+      @mode   = [mode].flatten
+      @command_prefix = ENV['BUNDLE_GEMFILE'] ? 'bundle exec ' : ''
+    end
+
+    def prepare!
+      # Remove the entire spec directory to make sure we have
+      # all the latest tests
+      FileUtils.rm_rf("#{@repo.tempdir}/spec")
+
+      # Remove any previous failure log
+      FileUtils.rm_f("#{@repo.tempdir}/failures.out")
+
+      # Create the other directories we need
+      FileUtils.mkdir_p("#{@repo.tempdir}/spec/classes")
+      FileUtils.mkdir_p("#{@repo.tempdir}/spec/acceptance/nodesets")
+
+      # Copy specified spec files over
+      @config.copy_spec_files(@repo)
+
+      # Create the Rakefile so that we can take advantage of the existing tasks
+      @config.write_rakefile(@repo.tempdir, "spec/classes/**/*_spec.rb")
+
+      # Create spec_helper.rb
+      @config.write_spec_helper("#{@repo.tempdir}/spec", @repo)
+
+      # Create spec_helper_accpetance.rb
+      @config.write_spec_helper_acceptance("#{@repo.tempdir}/spec", @repo)
+
+      if @mode.include?(:spec)
+        # Verify all of the spec tests
+        @config.spec_tests.each { |test| @config.verify_spec_test(@repo, test) }
+
+        # Deduplicate and write the tests (Spec and Acceptance)
+        @config.run_filters(PuppetlabsOnceover::Test.deduplicate(@config.spec_tests)).each do |test|
+          @config.write_spec_test("#{@repo.tempdir}/spec/classes", test)
+        end
+      end
+
+      if @mode.include?(:acceptance)
+        # Verify all of the acceptance tests
+        @config.acceptance_tests.each { |test| @config.verify_acceptance_test(@repo, test) }
+
+        # Write them out
+        @config.write_acceptance_tests(
+          "#{@repo.tempdir}/spec/acceptance",
+          @config.run_filters(PuppetlabsOnceover::Test.deduplicate(@config.acceptance_tests))
+        )
+      end
+
+      # Parse the current hiera config, modify, and write it to the temp dir
+      unless @repo.hiera_config == nil
+        hiera_config = @repo.hiera_config
+        hiera_config.each do |setting, value|
+          if value.is_a?(Hash)
+            if value.has_key?(:datadir)
+              hiera_config[setting][:datadir] = "#{@repo.tempdir}/#{@repo.environmentpath}/production/#{value[:datadir]}"
+            end
+          end
+        end
+        File.write("#{@repo.tempdir}/#{@repo.environmentpath}/production/hiera.yaml", hiera_config.to_yaml)
+      end
+
+      @config.create_fixtures_symlinks(@repo)
+    end
+
+    def run_spec!
+      Dir.chdir(@repo.tempdir) do
+        # Disable warnings unless we are running in debug mode
+        unless logger.level.zero?
+          previous_rubyopt = ENV.fetch('RUBYOPT', nil)
+          ENV['RUBYOPT']   = ENV['RUBYOPT'].to_s + ' -W0'
+        end
+
+        # NOTE: This is the way to provide options to rspec according to:
+        # https://github.com/puppetlabs/puppetlabs_spec_helper/blob/master/lib/puppetlabs_spec_helper/rake_tasks.rb#L51
+        ENV['CI_SPEC_OPTIONS'] = ENV['CI_SPEC_OPTIONS'].to_s + @config.filter_tags.map { |tag| " --tag #{tag}" }.join unless @config.filter_tags.nil?
+        ENV['CI_SPEC_OPTIONS'] = ENV['CI_SPEC_OPTIONS'].to_s + ' --fail-fast' if @config.fail_fast
+
+        if @config.opts[:parallel]
+          logger.debug "Running #{@command_prefix}rake parallel_spec from #{@repo.tempdir}"
+          result = run_command(@command_prefix.strip.split, 'rake', 'parallel_spec')
+        else
+          require 'io/console'
+          logger.debug "Running #{@command_prefix}rake spec:standalone from #{@repo.tempdir}"
+          result = run_command(@command_prefix.strip.split, 'rake', 'spec:standalone')
+        end
+
+        # Reset env to previous state if we modified it
+        unless logger.level.zero?
+          ENV['RUBYOPT'] = previous_rubyopt
+        end
+
+        # Print a summary if we were running in parallel
+        if @config.formatters.include? 'PuppetlabsOnceoverFormatterParallel'
+          require 'puppetlabs-onceover/rspec/formatters'
+          formatter = PuppetlabsOnceoverFormatterParallel.new(STDOUT)
+          formatter.output_results("#{repo.tempdir}/parallel")
+        end
+
+        # Finally exit and preserve the exit code
+        exit result.status.exitstatus
+      end
+    end
+
+    def run_acceptance!
+      warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
+
+      Dir.chdir(@repo.tempdir) do
+        #`bundle install --binstubs`
+        #`bin/rake spec:standalone`
+        logger.debug "Running #{@command_prefix}rake acceptance from #{@repo.tempdir}"
+        result = run_command(@command_prefix.strip.split, 'rake', 'acceptance')
+      end
+
+      # Finally exit and preserve the exit code
+      exit result.status.exitstatus
+    end
+
+    def run_command(*args)
+      begin
+        STDERR.raw! if STDERR.isatty
+        result = Backticks::Runner.new(interactive: true).run(args.flatten).join
+      ensure
+        STDERR.cooked! if STDERR.isatty
+      end
+    end
+  end
+end

data/lib/puppetlabs-onceover/test.rb

@@ -0,0 +1,135 @@
+class PuppetlabsOnceover
+  class Test
+    @@all =[]
+
+    attr_accessor :nodes
+    attr_accessor :classes
+    attr_accessor :test_config
+    attr_reader   :default_test_config
+    attr_reader   :tags
+
+    # This can accept a bunch of stuff. It can accept nodes, classes or groups anywhere
+    # it will then detect them and expand them out into their respective objects so that
+    # we just end up with a list of nodes and classes
+    def initialize(on_this, test_this, test_config)
+      @default_test_config = {
+        'check_idempotency' => true,
+        'runs_before_idempotency' => 1
+      }
+
+      # Add defaults if they do not exist
+      test_config = @default_test_config.merge(test_config)
+
+      @nodes   = []
+      @classes = []
+      @test_config = test_config
+      @test_config.delete('classes') # remove classes from the config
+
+      # Make sure that tags are an array
+      @test_config['tags'] ||= []
+      @test_config['tags'] = [@test_config['tags']].flatten
+      @tags = @test_config['tags']
+
+      # Get the nodes we are working on
+      if PuppetlabsOnceover::Group.find(on_this)
+        @nodes << PuppetlabsOnceover::Group.find(on_this).members
+      elsif PuppetlabsOnceover::Node.find(on_this)
+        @nodes << PuppetlabsOnceover::Node.find(on_this)
+      else
+        raise "#{on_this} was not found in the list of nodes or groups!"
+      end
+
+      @nodes.flatten!
+
+      # Check that our nodes list contains only nodes
+      raise "#{@nodes} contained a non-node object." unless @nodes.all? { |item| item.is_a?(PuppetlabsOnceover::Node) }
+
+      if test_this.is_a?(String)
+        # If we have just given a string then grab all the classes it corresponds to
+        if PuppetlabsOnceover::Group.find(test_this)
+          @classes << PuppetlabsOnceover::Group.find(test_this).members
+        elsif PuppetlabsOnceover::Class.find(test_this)
+          @classes << PuppetlabsOnceover::Class.find(test_this)
+        else
+          raise "#{test_this} was not found in the list of classes or groups!"
+        end
+        @classes.flatten!
+      elsif test_this.is_a?(Hash)
+        @classes = PuppetlabsOnceover::TestConfig.subtractive_to_list(test_this)
+      elsif test_this.is_a?(PuppetlabsOnceover::Class)
+        @classes << test_this
+      end
+    end
+
+    def eql?(other)
+      @nodes.sort.eql?(other.nodes.sort) and @classes.sort.eql?(other.classes.sort)
+    end
+
+    def to_s
+      class_msg = ""
+      node_msg  = ""
+      if classes.count > 1
+        class_msg = "#{classes.count}_classes"
+      else
+        class_msg = classes[0].name.gsub("::",'__')
+      end
+
+      if nodes.count > 1
+        node_msg = "#{nodes.count}_nodes"
+      else
+        node_msg = nodes[0].name
+      end
+
+      "#{class_msg}_on_#{node_msg}"
+    end
+
+    def self.deduplicate(tests)
+      require 'deep_merge'
+      # This should take an array of tests and remove any duplicates from them
+
+      # this will be an array of arrays, or maybe hashes
+      combinations = []
+      new_tests    = []
+
+      tests.each do |test|
+        test.nodes.each do |node|
+          test.classes.each do |cls|
+            combo = {node => cls}
+            if combinations.member?(combo)
+
+              # Find the right test object:
+              relevant_test = new_tests[new_tests.index do |a|
+                a.nodes[0] == node and a.classes[0] == cls
+              end]
+
+              # Delete all default values in the current options hash
+              test.test_config.delete_if do |key, value|
+                test.default_test_config[key] == value
+              end
+
+              # Merge the non-default options right on in there
+              relevant_test.test_config.deep_merge!(test.test_config)
+            else
+              combinations << combo
+              new_tests << PuppetlabsOnceover::Test.new(node, cls, test.test_config)
+            end
+          end
+        end
+      end
+
+      # The array that this returns should be ephemeral, it does not
+      # represent anything defined in a controlrepo and should just
+      # be passed into the thing doing the testing and then killed,
+      # we don't want too many copies of the same shit going around
+      #
+      # Actually based on the way things are written I don't think this
+      # will deduplicate node or class objects, just test objects,
+      # everything else is passed by reference
+      new_tests
+    end
+
+    def self.all
+      @@all
+    end
+  end
+end