onceover 3.11.1 → 3.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 0de0a768f2e22ad3164a270df1a526f814f6ddc21e8333cebca698e55cab3425
-  data.tar.gz: f1eca8504701aaedd2a84b5e949ad3defd013a190014f2276d93bde738216d2b
+SHA1:
+  metadata.gz: 3e2b22d1ea208cdaeeeb71b5a99e2f55f6056cb2
+  data.tar.gz: d29bad7de9cfd34c55ece9b69bbb73b5b48cb182
 SHA512:
-  metadata.gz: d06d28ae8e4bf7065ed9a25caa4b9f98dcab8fd5f825c84dccc501bce4274752cd35590b44d8c245b314b6b023f552a6a1adab0a084d9d2a109178b97df1baf5
-  data.tar.gz: 6bf9cb7ca6c63c9557e433e597e73b0e7a8c0c9cb1dda28552ca9fce366d6ecfff01c36854e34fe300b392c0d6ca91817f513bcb056ce2332d33d10c36441da9
+  metadata.gz: add796e1d03519fb411c5b2bc5a27b5ff0f00e52b9cea682aeb06b21ec31d36ddfcbbf5209a88fe9eda2a26650d49be47d611096f264b21e97302134faa7b8f0
+  data.tar.gz: 73ec3b013c0c02598fa04d99dee1721b91c40d40ec2bd174e24393177f3052ee084d222afbf8791bbaeda8ae0263679e2a88e5c53b78e31ad3728c934b142d55

data/Gemfile

@@ -2,6 +2,8 @@ source 'https://rubygems.org'
 
 gemspec
 
+gem 'pry-coolline', '> 0.0', '< 1.0.0'
+
 if ENV['PUPPET_VERSION']
   gem 'puppet', ENV['PUPPET_VERSION']
 end

data/features/formatting.feature

@@ -0,0 +1,36 @@
+@formatting
+Feature: Format errors nicely
+  I would like Onceover to format errors nicely so that they are easily read
+
+  Background:
+    Given the OnceoverFormatter
+
+  Scenario: Parse a missing class error
+    When Puppet throws the error: "error during compilation: Evaluation Error: Error while evaluating a Function Call, Could not find class ::role::websevrer for server01.foo.com (line: 17, column: 1) on node server01.foo.com"
+    Then the error should parse successfully
+    And it should find 1 error
+    And the parsed error should contain the following keys: text, line, column
+
+  Scenario: Parse an incorrect parameter error
+    When Puppet throws the error: "error during compilation: Evaluation Error: Error while evaluating a Resource Statement, Class[Docker]: has no parameter named 'package_name' (file: /some/path/tp/the/file/module/manifests/profile/docker.pp, line: 8, column: 1) on node server01.foo.com"
+    Then the error should parse successfully
+    And it should find 1 error
+    And the parsed error should contain the following keys: text, file, line, column
+
+  Scenario: Parse a duplication declaration error
+    When Puppet throws the error: "error during compilation: Evaluation Error: Error while evaluating a Resource Statement, Duplicate declaration: Package[virtualenv] is already declared at (file: /some/path/tp/the/file/foo/manifests/profile/ti2.pp, line: 31); cannot redeclare (file: /some/path/tp/the/file/python/manifests/install.pp, line: 54) (file: /some/path/tp/the/file/python/manifests/install.pp, line: 54, column: 3) on node server01.foo.co"
+    Then the error should parse successfully
+    And it should find 2 errors
+    And the parsed errors should contain the following keys: text, file, line, column
+
+  Scenario: Parse a lookup failure error
+    When Puppet throws the error: "error during compilation: Function lookup() did not find a value for the name 'archvsync::repositories' on node server01.foo.com"
+    Then the error should parse successfully
+    And it should find 1 errors
+    And the parsed errors should contain the following keys: text
+
+  Scenario: Fail to parse an error and survive anyway
+    When Puppet throws the error: "Derp derp derp. This error is 💩"
+    Then the error should parse successfully
+    And it should find 1 errors
+    And the parsed errors should contain the following keys: text

data/features/step_definitions/formatter.rb

@@ -0,0 +1,41 @@
+Given(/^the OnceoverFormatter$/) do
+  require 'rspec'
+  require 'onceover/rspec/formatters'
+
+  RSpec.configure do |c|
+    # Create onceover settings to be accessed by formatters
+    c.add_setting :onceover_tempdir
+    c.add_setting :onceover_root
+    c.add_setting :onceover_environmentpath
+  
+    c.onceover_tempdir         = "/Users/foo/git/controlrepo/.onceover"
+    c.onceover_root            = "/Users/foo/git/controlrepo"
+    c.onceover_environmentpath = "etc/puppetlabs/code/environments"
+  end
+  
+  @formatter = OnceoverFormatter.new(STDOUT)
+end
+
+When(/^Puppet throws the error: "(.*)"$/) do |error|
+  @error = error
+end
+
+Then(/^the error should parse successfully$/) do
+  expect do
+    @parsed_error = @formatter.parse_errors(@error)
+  end.to_not raise_error
+end
+
+Then(/^it should find (\d+) errors?$/) do |number|
+  expect(@parsed_error.length).to be(number.to_i)
+end
+
+Then(/^the parsed errors? should contain the following keys: (.*)$/) do |keys|
+  # Split the keys into an array
+  keys = keys.split(',').map(&:strip)
+  @parsed_error.each do |error|
+    keys.each do |k|
+      expect(error).to have_key(k.to_sym)
+    end
+  end
+end

data/lib/onceover/cli/run.rb

@@ -41,7 +41,7 @@ This includes deploying using r10k and running all custom tests.
             summary 'Runs spec tests'
 
             optional :p, :parallel, 'Runs spec tests in parallel. This increases speed at the cost of poorly formatted logs and irrelevant junit output.'
-            optional nil, :format, 'Which RSpec formatter to use, valid options are: documentation, progress, FailureCollector. You also specify this multiple times', multiple: true, default: :defaults
+            optional nil, :format, 'Which RSpec formatter to use, valid options are: documentation, progress, FailureCollector, OnceoverFormatter. You also specify this multiple times', multiple: true, default: :defaults
 
             run do |opts, args, cmd|
               repo = Onceover::Controlrepo.new(opts)

data/lib/onceover/controlrepo.rb

@@ -297,7 +297,7 @@ class Onceover
       threads.map(&:join)
       puppetfile_string = queue.pop
 
-      File.open(@puppetfile, 'w') {|f| f.write(puppetfile_string.join("\n")) }
+      File.open(@puppetfile, 'w') {|f| f.puts(puppetfile_string.join("\n")) }
       puts "#{'changed'.yellow} #{@puppetfile}"
     end
 

data/lib/onceover/rspec/formatters.rb

@@ -0,0 +1,239 @@
+require 'pathname'
+
+class OnceoverFormatter
+  RSpec::Core::Formatters.register self, :example_group_started,
+    :example_passed, :example_failed, :example_pending, :dump_failures#, :dump_summary
+
+  COMPILATION_ERROR      = %r{error during compilation: (?<error>.*)}
+  ERROR_WITH_LOCATION    = %r{(?<error>.*?)\s(at )?(\((file: (?<file>.*?), )?line: (?<line>\d+)(, column: (?<column>\d+))?\))(; )?}
+  ERROR_WITHOUT_LOCATION = %r{(?<error>.*?)\son node}
+
+  def initialize output
+    @output        = output
+    @previous_role = nil
+  end
+
+  def example_group_started notification
+    if notification.group.parent_groups == [notification.group]
+      # If this is the highest level group (The role)
+      role = notification.group.description
+      if role != @previous_role
+        @output << "\n"
+        @output << class_name("#{notification.group.description}:")
+
+        # Calculate the padding required
+        padding = (longest_group - role.length) + 1
+        # Create padding
+        padding.times { @output << ' ' }
+
+        # Save the role name
+        @previous_role = role
+      end
+    else
+      # If not then this will be a test for that role
+      @output << '? '
+    end
+  end
+
+  def example_passed notification
+    @output << "\b\b"
+    @output << "#{green('P')} "
+  end
+
+  def example_failed notification
+    @output << "\b\b"
+    @output << "#{red('F')} "
+  end
+
+  def example_pending notification
+    @output << "\b\b"
+    @output << "#{yellow('?')} "
+  end
+
+  def dump_failures notification
+    require 'onceover/controlrepo'
+
+    # Group by role
+    grouped = notification.failed_examples.group_by { |e| e.metadata[:example_group][:parent_example_group][:description]}
+
+    # Further group by error
+    grouped.each do |role, failures|
+      grouped[role] = failures.uniq { |f| f.metadata[:execution_result].exception.to_s }
+    end
+
+    # Put some spacing before the results
+    @output << "\n\n\n"
+
+    grouped.each do |role, failures|
+      role = {
+        name: role,
+        errors: failures.map { |f| parse_errors(f.metadata[:execution_result].exception.to_s)}.flatten,
+      }
+
+      @output << Onceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding)
+    end
+  end
+
+  def parse_errors(raw_error)
+    # Check if the error is a compilation error
+    match = COMPILATION_ERROR.match(raw_error)
+    if match
+      compilation_error = match['error']
+      # Check if we car parse it
+      if ERROR_WITH_LOCATION.match(compilation_error)
+        scanned_errors = match['error'].scan(ERROR_WITH_LOCATION)
+
+        # Delete any matches where there was no error text
+        scanned_errors.delete_if { |e| e.first.empty? }
+
+        scanned_errors.map do |error_matches|
+          {
+            text:   error_matches[0],
+            file:   calculate_relative_source(error_matches[1]),
+            line:   error_matches[2],
+            column: error_matches[3],
+          }
+        end
+      elsif ERROR_WITHOUT_LOCATION.match(compilation_error)
+        scanned_errors = match['error'].scan(ERROR_WITHOUT_LOCATION)
+
+        # Delete any matches where there was no error text
+        scanned_errors.delete_if { |e| e.first.empty? }
+
+        scanned_errors.map do |error_matches|
+          {
+            text: error_matches[0],
+          }
+        end
+      else
+        [{
+          text: raw_error,
+        }]
+      end
+    else
+      [{
+        text: raw_error,
+      }]
+    end
+  end
+
+  # This method calculates where the original source file is relative to the
+  # user's current location. This is more compliacted than it sounds because
+  # if we are running from the root of the controlrepo and we have an error in:
+  #
+  # /Users/dylan/git/puppet_controlrepo/.onceover/etc/puppetlabs/code/environments/production/site/role/manifests/lb.pp
+  #
+  # We need that to end up pointing at the original source file not the cached
+  # one i.e.
+  #
+  # site/role/manifests/lb.pp
+  #
+  def calculate_relative_source(file)
+    return nil if file.nil?
+
+    file            = Pathname.new(file)
+    tempdir         = Pathname.new(RSpec.configuration.onceover_tempdir)
+    root            = Pathname.new(RSpec.configuration.onceover_root)
+    environmentpath = Pathname.new(RSpec.configuration.onceover_environmentpath)
+
+    # Calculate the full relative path
+    file.relative_path_from(tempdir + environmentpath + "production").to_s
+  end
+
+  private
+
+  # Below are defined the styles for the output
+  def class_name(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :bold)
+  end
+
+  def black(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :black)
+  end
+
+  def red(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :red)
+  end
+
+  def green(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :green)
+  end
+
+  def yellow(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :yellow)
+  end
+
+  def blue(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :blue)
+  end
+
+  def magenta(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :magenta)
+  end
+
+  def cyan(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :cyan)
+  end
+
+  def white(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :white)
+  end
+
+  def bold(text)
+    RSpec::Core::Formatters::ConsoleCodes.wrap(text, :bold)
+  end
+
+  def longest_group
+    RSpec.configuration.world.example_groups.max { |a,b| a.description.length <=> b.description.length}.description.length
+  end
+
+end
+
+# class OnceoverFormatterParallel < OnceoverFormatter
+#   require 'yaml'
+
+#   def example_group_started notification
+#     # Do nothing
+#   end
+
+#   def example_passed notification
+#     @output << green('P')
+#   end
+
+#   def example_failed notification
+#     @output << red('F')
+#   end
+
+#   def example_pending notification
+#     @output << yellow('?')
+#   end
+
+#   def dump_failures
+#     # TODO: This should write to a file and then get picked up and formatted by onceover itself
+#     # might need to use a module for the formatting
+#     require 'pry'
+#     binding.pry
+#     RSpec.configuration.onceover_tempdir
+#   end
+
+# end
+
+class FailureCollector
+  RSpec::Core::Formatters.register self, :dump_failures
+
+  def initialize(output)
+    FileUtils.touch(File.expand_path("#{RSpec.configuration.onceover_tempdir}/failures.out"))
+  end
+
+  def dump_failures(failures)
+    open(File.expand_path("#{RSpec.configuration.onceover_tempdir}/failures.out"), 'a') { |f|
+      failures.failed_examples.each do |fe|
+        f.puts
+        f.puts "#{fe.metadata[:description]}"
+        f.puts "#{fe.metadata[:execution_result].exception.to_s}"
+        f.puts "#{fe.metadata[:file_path]}:#{fe.metadata[:line_number]}"
+        f.puts "------------------------------------------------------"
+      end
+    }
+  end
+end

data/lib/onceover/testconfig.rb

@@ -53,7 +53,7 @@ class Onceover
       
       # Set dynamic defaults for format
       if opts[:format] == [:defaults]
-        @formatters = opts[:parallel] ? ['documentation', 'FailureCollector'] : ['documentation']
+        @formatters = opts[:parallel] ? ['documentation', 'FailureCollector'] : ['OnceoverFormatter']
       else
         @formatters = opts[:format]
       end

data/onceover.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "onceover"
-  s.version     = "3.11.1"
+  s.version     = "3.12.0"
   s.authors     = ["Dylan Ratcliffe"]
   s.email       = ["dylan.ratcliffe@puppet.com"]
   s.homepage    = "https://github.com/dylanratcliffe/onceover"

data/templates/error_summary.yaml.erb

@@ -0,0 +1,14 @@
+<%= bold(role[:name]) %>: <%= red('failed') %>
+  errors:
+<% role[:errors].each do |error| -%>
+    <%= red(error[:text]) %>
+<% if error[:file] -%>
+      file: <%= bold(error[:file]) %>
+<% end -%>
+<% if error[:line] -%>
+      line: <%= bold(error[:line]) %>
+<% end -%>
+<% if error[:column] -%>
+      column: <%= bold(error[:column]) %>
+<% end -%>
+<% end -%>

data/templates/spec_helper.rb.erb

@@ -14,30 +14,18 @@ end
 
 require 'puppetlabs_spec_helper/module_spec_helper'
 require 'rspec_junit_formatter'
+require 'onceover/rspec/formatters'
 
-class FailureCollector
-  RSpec::Core::Formatters.register self, :dump_failures
-
-  def initialize(output)
-    @output = output
-    puts "Initializing FailureCollector"
-    FileUtils.touch(File.expand_path('<%= repo.tempdir %>/failures.out'))
-  end
+RSpec.configure do |c|
+  # Create onceover settings to be accessed by formatters
+  c.add_setting :onceover_tempdir
+  c.add_setting :onceover_root
+  c.add_setting :onceover_environmentpath
 
-  def dump_failures(failures)
-    open(File.expand_path('<%= repo.tempdir %>/failures.out'), 'a') { |f|
-      failures.failed_examples.each do |fe|
-        f.puts
-        f.puts "#{fe.metadata[:description]}"
-        f.puts "#{fe.metadata[:execution_result].exception.to_s}"
-        f.puts "#{fe.metadata[:file_path]}:#{fe.metadata[:line_number]}"
-        f.puts "------------------------------------------------------"
-      end
-    }
-  end
-end
+  c.onceover_tempdir         = <%= repo.tempdir.inspect %>
+  c.onceover_root            = <%= repo.root.inspect %>
+  c.onceover_environmentpath = <%= repo.environmentpath.inspect %>
 
-RSpec.configure do |c|
   # Also add JUnit output in case people want to use that
   c.add_formatter('RSpecJUnitFormatter','<%= repo.tempdir %>/spec.xml')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: onceover
 version: !ruby/object:Gem::Version
-  version: 3.11.1
+  version: 3.12.0
 platform: ruby
 authors:
 - Dylan Ratcliffe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-26 00:00:00.000000000 Z
+date: 2019-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -343,12 +343,14 @@ files:
 - factsets/solaris-11.2-sparc-64.json
 - factsets/windows-10-64.json
 - features/cache.feature
+- features/formatting.feature
 - features/help.feature
 - features/init.feature
 - features/run.feature
 - features/show.feature
 - features/step_definitions/cache.rb
 - features/step_definitions/common.rb
+- features/step_definitions/formatter.rb
 - features/step_definitions/init.rb
 - features/step_definitions/run.rb
 - features/support/cache_helper.rb
@@ -370,6 +372,7 @@ files:
 - lib/onceover/logger.rb
 - lib/onceover/node.rb
 - lib/onceover/rake_tasks.rb
+- lib/onceover/rspec/formatters.rb
 - lib/onceover/runner.rb
 - lib/onceover/test.rb
 - lib/onceover/testconfig.rb
@@ -423,6 +426,7 @@ files:
 - templates/Rakefile.erb
 - templates/acceptance_test_spec.rb.erb
 - templates/controlrepo.yaml.erb
+- templates/error_summary.yaml.erb
 - templates/factsets_README.md.erb
 - templates/nodeset.yaml.erb
 - templates/pre_conditions_README.md.erb
@@ -449,7 +453,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubyforge_project: 
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: Testing tools for Puppet controlrepos