ceedling 1.1.10 → 1.2.1

data/plugins/report_tests_log_factory/lib/json_tests_reporter.rb

@@ -18,10 +18,10 @@ class JsonTestsReporter < TestsReporter
     hash = {
       "Name"          => name,
       "BuildDuration" => duration_s,
-      "FailedTests"   => write_failures( results[:failures] ),
-      "PassedTests"   => write_tests( results[:successes] ),
-      "IgnoredTests"  => write_tests( results[:ignores] ),
-      "Summary"       => write_statistics( results[:counts] )
+      "FailedTests"   => failure_entries( results[:failures] ),
+      "PassedTests"   => test_entries( results[:successes] ),
+      "IgnoredTests"  => test_entries( results[:ignores] ),
+      "Summary"       => statistics( results[:counts] )
     }
 
     stream << JSON.pretty_generate(hash)
@@ -31,44 +31,31 @@ class JsonTestsReporter < TestsReporter
 
   private
 
-  def write_failures(results)
-    # Array of hashes relating a source file, test, and test failure
-    failures = []
-    results.each do |result|
-      result[:collection].each do |item|
-        failures << {
-          "file" => result[:source][:file],
-          "test" => item[:test],
-          "line" => item[:line],
-          "message" => item[:message]
-        }
-      end
+  # Each failure relates a source file, test, line, and message
+  def failure_entries(results)
+    entries = tests_in( results ).map do |file, item|
+      { "file" => file, "test" => item[:test], "line" => item[:line], "message" => item[:message] }
     end
-    return failures.uniq
+    return entries.uniq
   end
 
-  def write_tests(results)
-    # Array of hashes relating a source file and test
-    successes = []
-    results.each do |result|
-      result[:collection].each do |item|
-        successes << { 
-          "file" => result[:source][:file],
-          "test" => item[:test]
-        }
-      end
-    end
-    return successes
+  # Each entry relates a source file and test
+  def test_entries(results)
+    return tests_in( results ).map { |file, item| { "file" => file, "test" => item[:test] } }
+  end
+
+  # Every test in a results category, paired with its source file
+  def tests_in(results)
+    return results.flat_map { |result| result[:collection].map { |item| [ result[:source][:file], item ] } }
   end
 
-  def write_statistics(counts)
-    # Hash of keys:values for statistics
+  def statistics(counts)
     return {
       "total_tests" => counts[:total],
-      "passed" => (counts[:total] - counts[:ignored] - counts[:failed]),
-      "ignored" => counts[:ignored],
-      "failures" => counts[:failed]
+      "passed"      => counts[:passed],
+      "ignored"     => counts[:ignored],
+      "failures"    => counts[:failed]
     }
   end
 
-end
+end

data/plugins/report_tests_log_factory/lib/junit_tests_reporter.rb

@@ -9,29 +9,33 @@ require 'tests_reporter'
 
 class JunitTestsReporter < TestsReporter
 
+  # :file_line_anchor writes a located test's file attribute as path#line, a form some
+  # tools follow to the line
   def setup()
     super( default_filename: 'junit_tests_report.xml' )
+    @file_line_anchor = fetch_config_value( :file_line_anchor ) == true
   end
 
-  def header(stream:, name:, results:, duration_s:)
+  # Each results category and the outcome its tests carry in a suite
+  CATEGORIES = { successes: :success, failures: :failed, ignores: :ignored }.freeze
+
+  def header(stream:, name:, results:, **)
     stream.puts( '<?xml version="1.0" encoding="utf-8" ?>' )
     stream.puts(
-      '<testsuites name="%s" '     % name +
+      '<testsuites name="%s" '     % xml_escape( name ) +
                   'tests="%d" '    % results[:counts][:total] +
                   'failures="%d" ' % results[:counts][:failed] +
                   'time="%.3f">'   % results[:total_time]
     )
   end
 
-  def body(stream:, name:, results:, duration_s:)
-    suites = reorganize_results( results )
-
-    suites.each do |suite|
+  def body(stream:, results:, **)
+    reorganize_results( results ).each do |suite|
       write_suite( suite, stream )
     end
   end
 
-  def footer(stream:, name:, results:, duration_s:)
+  def footer(stream:, **)
     stream.puts( '</testsuites>' )
   end
 
@@ -39,160 +43,114 @@ class JunitTestsReporter < TestsReporter
 
   private
 
-  # Reorganize test results by test executable instead of by result category
-  # Original success structure: successeses { file => test_cases[] }
-  # Reorganized test results:   file => test_cases[{... result: :success}]
-  def reorganize_results( results )
-    # Create structure of hash with default values
-    suites = Hash.new() do |h,k|
-      h[k] = {
-        collection: [],
-        total:   0,
-        success: 0,
-        failed:  0,
-        ignored: 0,
-        errors:  0,
-        time:    0,
-        stdout:  []
-      }
-    end
-
-    results[:successes].each do |result|
-      # Extract filepath
-      source = result[:source][:file]
+  # Regroups results by test file, the unit a JUnit testsuite describes, instead of by
+  # outcome. Every test carries its outcome, and a suite's counts come from its tests.
+  # Results are read, never altered, since every configured reporter shares them.
+  def reorganize_results(results)
+    suites = Hash.new { |hash, name| hash[name] = { name: name, collection: [], time: 0, stdout: [] } }
 
-      # Filepath minus file extension
-      name = source.sub( /#{File.extname(source)}$/, '' )
+    CATEGORIES.each do |category, outcome|
+      results[category].each { |result| add_tests( suites, result, results[:times], outcome ) }
+    end
 
-      # Sanitize: Ensure no nil elements
-      result[:collection].compact!
+    results[:stdout].each { |result| add_stdout( suites, result ) }
 
-      # Sanitize: Ensure no empty test result hashes
-      result[:collection].select! {|test| !test.empty?() }
+    return suites.values
+  end
 
-      # Add success test cases to full test case collection and update statistics
-      suites[name][:collection] += result[:collection].map{|test| test.merge(result: :success)}
-      suites[name][:total] += result[:collection].length
-      suites[name][:success] += result[:collection].length
-      suites[name][:time] = results[:times][source]
-    end
+  def add_tests(suites, result, times, outcome)
+    source = result[:source][:file]
+    suite  = suites[suite_name( source )]
 
-    results[:failures].each do |result|
-      # Extract filepath
-      source = result[:source][:file]
+    # Skip nil and empty entries a results file may carry
+    tests = result[:collection].reject { |test| test.nil? or test.empty? }
 
-      # Filepath minus file extension
-      name = source.sub( /#{File.extname(source)}$/, '' )
+    suite[:collection] += tests.map { |test| test.merge( result: outcome, source: source ) }
+    suite[:time] = times[source]
+  end
 
-      # Sanitize: Ensure no nil elements
-      result[:collection].compact!
+  def add_stdout(suites, result)
+    suites[suite_name( result[:source][:file] )][:stdout] += result[:collection]
+  end
 
-      # Sanitize: Ensure no empty test result hashes
-      result[:collection].select! {|test| !test.empty?() }
+  # A suite is named for its test file without the file's extension
+  def suite_name(source)
+    return source.delete_suffix( File.extname( source ) )
+  end
 
-      # Add failure test cases to full test case collection and update statistics
-      suites[name][:collection] += result[:collection].map{|test| test.merge(result: :failed)}
-      suites[name][:total] += result[:collection].length
-      suites[name][:failed] += result[:collection].length
-      suites[name][:time] = results[:times][source]
-    end
+  def write_suite(suite, stream)
+    stream.puts( suite_tag( suite ) )
+    suite[:collection].each { |test| write_test( test, stream ) }
+    write_stdout( suite[:stdout], stream )
+    stream.puts( '  </testsuite>' )
+  end
 
-    results[:ignores].each do |result|
-      # Extract filepath
-      source = result[:source][:file]
+  # Ceedling has no notion of a test error distinct from a failure, so errors is always 0
+  def suite_tag(suite)
+    tests   = suite[:collection]
+    failed  = tests.count { |test| test[:result] == :failed }
+    ignored = tests.count { |test| test[:result] == :ignored }
 
-      # Filepath minus file extension
-      name = source.sub( /#{File.extname(source)}$/, '' )
+    return '  <testsuite name="%s" tests="%d" failures="%d" skipped="%d" errors="0" time="%.3f">' %
+      [ xml_escape( suite[:name] ), tests.length, failed, ignored, suite[:time] ]
+  end
 
-      # Sanitize: Ensure no nil elements
-      result[:collection].compact!
+  def write_stdout(lines, stream)
+    return if lines.empty?
 
-      # Sanitize: Ensure no empty test result hashes
-      result[:collection].select! {|test| !test.empty?() }
+    stream.puts( '    <system-out>' )
+    lines.each { |line| stream.puts( xml_escape( line ) ) }
+    stream.puts( '    </system-out>' )
+  end
 
-      # Add ignored test cases to full test case collection and update statistics
-      suites[name][:collection] += result[:collection].map{|test| test.merge(result: :ignored)}
-      suites[name][:total] += result[:collection].length
-      suites[name][:ignored] += result[:collection].length
-      suites[name][:time] = results[:times][source]
-    end
+  # A passing test is an empty element. Any other test holds an element for its outcome.
+  def write_test(test, stream)
+    opening = "    <testcase #{attributes( testcase_attributes( test ) )}"
+    outcome = outcome_element( test )
 
-    results[:stdout].each do |result|
-      # Extract filepath
-      source = result[:source][:file]
-      # Filepath minus file extension
-      name = source.sub( /#{File.extname(source)}$/, '' )
+    return stream.puts( opening + '/>' ) if outcome.nil?
 
-      # Add $stdout messages to collection
-      suites[name][:stdout] += result[:collection]
-    end
+    stream.puts( opening + '>' )
+    stream.puts( '      ' + outcome )
+    stream.puts( '    </testcase>' )
+  end
 
-    # Add name to suite hashes (duplicating the key for suites)
-    suites.map{|name, data| data.merge(name: name) }
+  # A test's class is its suite, path kept, so same-named test files in different folders,
+  # and their same-named test cases, stay distinct where CI tools group results by class.
+  # A test carries the line Unity reported for it, when its results name one.
+  def testcase_attributes(test)
+    attributes = { name: test[:test], classname: suite_name( test[:source] ), file: test_file( test ) }
+    attributes[:line] = test[:line] unless test[:line].nil?
+    attributes[:time] = format( '%.3f', test[:unity_test_time] )
+    return attributes
   end
 
-  def write_suite( suite, stream )
-    stream.puts(
-      '  <testsuite name="%s" '     % suite[:name] +
-                   'tests="%d" '    % suite[:total] +
-                   'failures="%d" ' % suite[:failed] +
-                   'skipped="%d" '  % suite[:ignored] +
-                   'errors="%d" '   % suite[:errors] +
-                   'time="%.3f">'   % suite[:time]
-    )
+  def test_file(test)
+    return test[:source] unless @file_line_anchor && test[:line]
 
-    suite[:collection].each do |test|
-      write_test( test, stream )
-    end
+    return "#{test[:source]}##{test[:line]}"
+  end
 
-    unless suite[:stdout].empty?
-      stream.puts('    <system-out>')
-      suite[:stdout].each do |line|
-        line.gsub!(/&/, '&amp;')
-        line.gsub!(/</, '&lt;')
-        line.gsub!(/>/, '&gt;')
-        line.gsub!(/"/, '&quot;')
-        line.gsub!(/'/, '&apos;')
-        stream.puts( line )
-      end
-      stream.puts('    </system-out>')
-    end
+  # A failure's message starts with its file and line in the form compilers use, which
+  # readers and tools recognize
+  def located_message(test)
+    return test[:message] if test[:line].nil?
 
-    stream.puts('  </testsuite>')
+    return "#{test[:source]}:#{test[:line]}: #{test[:message]}"
   end
 
-  def write_test( test, stream )
-    test[:test].gsub!(/&/, '&amp;')
-    test[:test].gsub!(/</, '&lt;')
-    test[:test].gsub!(/>/, '&gt;')
-    test[:test].gsub!(/"/, '&quot;')
-    test[:test].gsub!(/'/, '&apos;')
+  # Each attribute's value escaped, in the order given
+  def attributes(pairs)
+    return pairs.map { |key, value| %(#{key}="#{xml_escape( value )}") }.join( ' ' )
+  end
 
+  def outcome_element(test)
     case test[:result]
-    when :success
-      stream.puts(
-        '    <testcase name="%<test>s" '               % test +
-                      'time="%<unity_test_time>.3f"/>' % test
-      )
-
     when :failed
-      stream.puts(
-        '    <testcase name="%<test>s" '              % test +
-                      'time="%<unity_test_time>.3f">' % test
-      )
-
-      if test[:message].empty?
-        stream.puts( '      <failure />' )
-      else
-        stream.puts( '      <failure message="%s" />' % test[:message] )
-      end
-      
-      stream.puts( '    </testcase>' )
-
+      return '<failure />' if test[:message].to_s.empty?
+      return '<failure message="%s" />' % xml_escape( located_message( test ) )
     when :ignored
-      stream.puts( '    <testcase name="%<test>s" time="%<unity_test_time>.3f">' % test )
-      stream.puts( '      <skipped />' )
-      stream.puts( '    </testcase>' )
+      return '<skipped />'
     end
   end
 end

data/plugins/report_tests_log_factory/lib/report_tests_log_factory.rb

@@ -6,30 +6,22 @@
 # =========================================================================
 
 require 'ceedling/plugins/plugin'
+require 'tests_reporter'
 
 class ReportTestsLogFactory < Plugin
 
-  DEFAULT_REPORT_NAME = "Ceedling Test Suite"
-
   TestBuild = Struct.new( :results_filepaths, :start_time_s, :end_time_s )
 
   # `Plugin` setup()
   def setup
     # Hash: Context symbol => TestBuild struct
     @build_results = {}
-    
-    # Get our test suite reports' configuration
-    config = @ceedling[:setupinator].config_hash
-    @config = config[:report_tests_log_factory]
-    
-    # Get list of enabled reports
-    reports = @config[:reports]
 
-    # Array of Reporter subclass objects
-    @reporters = load_reporters( reports, @config )
+    config     = @ceedling[:setupinator].config_hash[:report_tests_log_factory]
+    @reporters = load_reporters( config[:reports], config )
 
-    # Disable this plugin if no reports configured
-    @enabled = !(reports.empty?)
+    # With no reports configured, every hook does nothing
+    @enabled = !@reporters.empty?
 
     @mutex = Mutex.new()
 
@@ -53,8 +45,8 @@ class ReportTestsLogFactory < Plugin
     context = arg_hash[:context]
 
     @mutex.synchronize do
-      # Create a TestBuild entry if pre_test_build did not fire for this context.
-      # `ceedling summary` will not fire pre_test_build.
+      # Fixtures run inside a test build, so pre_test_build has normally made this entry.
+      # Without one, the results are still reported, with no duration.
       @build_results[context] ||= TestBuild.new( [], nil, nil )
       @build_results[context].results_filepaths << arg_hash[:result_file]
     end
@@ -74,98 +66,92 @@ class ReportTestsLogFactory < Plugin
     return if not @enabled
     return if @build_results.empty?
 
-    msg = @reportinator.generate_heading( "Running Test Suite Reports" )
-    @loginator.log( msg )
-
-    # For each configured reporter, generate a test suite report per test context
-    @build_results.each do |context, test_build|
-      # Assemble results from all results filepaths collected
-      _results = @ceedling[:plugin_reportinator].assemble_test_results( test_build.results_filepaths )
-
-      # Provide results to each Reporter
-      @reporters.each do |reporter|
-        filepath = File.join( PROJECT_BUILD_ARTIFACTS_ROOT, context.to_s, reporter.filename )
-        name = generate_report_name( context )
-
-        msg = @reportinator.generate_progress( "Generating artifact #{filepath}" )
-        @loginator.log( msg )
-
-        start_s    = test_build.start_time_s
-        end_s      = test_build.end_time_s
-        duration_s = (start_s && end_s) ? (end_s - start_s) : nil
-        reporter.write(
-          name: name,
-          filepath: filepath,
-          results: _results,
-          duration_s: duration_s
-        )
+    reporting do
+      @build_results.each do |context, test_build|
+        results = @ceedling[:plugin_reportinator].assemble_test_results( test_build.results_filepaths )
+        write_reports( context, results, duration_s( test_build ) )
       end
     end
-
-  # White space at command line after all progress messages
-  @loginator.log( '' )
   end
 
-  ### Private
+  # `Plugin` summary hook -- report on the test results an earlier build left on disk.
+  # No build ran, so there is no duration. Only the test context's results are read.
+  def summary
+    return if not @enabled
 
-  private
+    result_list = @ceedling[:file_path_utils].form_pass_results_filelist( PROJECT_TEST_RESULTS_PATH, COLLECTION_ALL_TESTS )
+    results     = @ceedling[:plugin_reportinator].assemble_test_results( result_list, {:boom => false} )
 
-  def generate_report_name(context)
-      # Resolve and inject the display name used in report titles/headers.
+    reporting { write_reports( TEST_SYM, results, nil ) }
+  end
 
-      # Start with project name from configuration
-      _name = @ceedling[:configurator].project_name.to_s.strip
+  # The class a configured report loads by convention, e.g. 'fancy_shmancy' loads
+  # FancyShmancyTestsReporter
+  def self.reporter_class_name(report)
+    camel = report.gsub( /_./ ) { |match| match.upcase.delete( '_' ) }
+    return camel[0].capitalize + camel[1..] + 'TestsReporter'
+  end
 
-      # Default to a generic name if project name is empty
-      name = _name.empty? ? DEFAULT_REPORT_NAME : _name
+  ### Private
 
-      # Prepend name with context if not the default test context (TEST_SYM)
-      if context != TEST_SYM
-        name = "[#{context.to_s.upcase}] #{name}"
-      end
+  private
 
-      return name
+  # Frames report generation's progress messages with a heading and trailing white space
+  def reporting
+    @loginator.log( @reportinator.generate_heading( "Running Test Suite Reports" ) )
+    yield
+    @loginator.log( '' )
   end
 
-  def load_reporters(reports, config)
-    reporters = []
-
-    # For each report name string in configuration, dynamically load the corresponding 
-    # Reporter subclass by convention
+  # Writes every configured report for one context under that context's artifacts directory
+  def write_reports(context, results, duration_s)
+    name = generate_report_name( context )
 
-    # The steps below limit the set up complexity that would otherwise be
-    # required of a user's custom Reporter subclass
-    reports.each do |report|
-      # Enforce lowercase convention internally
-      report = report.downcase()
+    @reporters.each do |reporter|
+      filepath = File.join( PROJECT_BUILD_ARTIFACTS_ROOT, context.to_s, reporter.filename )
+      @loginator.log( @reportinator.generate_progress( "Generating artifact #{filepath}" ) )
+      reporter.write( name: name, filepath: filepath, results: results, duration_s: duration_s )
+    end
+  end
 
-      # Convert report configuration name 'foo_bar' to 'FooBarTestReporter' class name
-      #  1. Convert 'x_Y' (snake case) to camel case ('xY')
-      #  2. Capitalize first character of config name and add rest of class name
-      _reporter = report.gsub(/_./) {|match| match.upcase().delete('_') }
-      _reporter = _reporter[0].capitalize() + _reporter[1..-1] + 'TestsReporter'
+  # A build's duration, or nil when either end of it went unrecorded
+  def duration_s(test_build)
+    start_s = test_build.start_time_s
+    end_s   = test_build.end_time_s
+    return (start_s && end_s) ? (end_s - start_s) : nil
+  end
 
-      # Load each Reporter sublcass Ruby file dynamically by convention
-      # For custom user subclasses, requires directoy in :plugins ↳ :load_paths
-      require "#{report}_tests_reporter"
+  # The display name in report titles and headers. A context other than the default test
+  # context prefixes it.
+  def generate_report_name(context)
+    name = @ceedling[:configurator].project_name.to_s.strip
+    name = TestsReporter::DEFAULT_REPORT_NAME if name.empty?
 
-      # Dynamically instantiate Reporter subclass object
-      reporter = eval( "#{_reporter}.new(handle: :#{report})" )
+    return name if context == TEST_SYM
+    return "[#{context.to_s.upcase}] #{name}"
+  end
 
-      # Inject configuration
-      reporter.config = config[report.to_sym]
+  # Each configured report loads its Reporter subclass by naming convention. The factory
+  # injects configuration and utilities, which keeps a custom subclass's own setup small.
+  # A report named twice is written once.
+  def load_reporters(reports, config)
+    return reports.map( &:downcase ).uniq.map { |report| load_reporter( report, config[report.to_sym] ) }
+  end
 
-      # Inject utilty object
-      reporter.config_walkinator = @ceedling[:config_walkinator]
+  def load_reporter(report, config)
+    # A custom subclass's directory must be in :plugins ↳ :load_paths
+    require "#{report}_tests_reporter"
 
-      # Perform Reporter sublcass set up
-      reporter.setup()
+    # A custom subclass that breaks the naming convention fails with a NameError naming
+    # the missing class
+    reporter = Object.const_get( ReportTestsLogFactory.reporter_class_name( report ) ).new( handle: report.to_sym )
 
-      # Add new object to our internal list
-      reporters << reporter
-    end
+    reporter.config            = config
+    reporter.config_walkinator = @ceedling[:config_walkinator]
+    reporter.file_wrapper      = @ceedling[:file_wrapper]
+    reporter.setup()
 
-    return reporters
+    return reporter
   end
 
 end

data/plugins/report_tests_log_factory/lib/tests_reporter.rb

@@ -5,24 +5,32 @@
 #   SPDX-License-Identifier: MIT
 # =========================================================================
 
+require 'cgi'
+require 'stringio'
+
 class TestsReporter
 
+  # The report name when a project names none. A report may title itself differently
+  # for it, so it is shared here.
+  DEFAULT_REPORT_NAME = 'Ceedling Test Suite'
+
   # Dependency injection
   attr_writer :config_walkinator
 
   # Setup value injection
   attr_writer :config
 
+  # Dependency injection. A report renders in memory and reaches disk only through this,
+  # so a test can exercise a whole report without touching the filesystem.
+  attr_writer :file_wrapper
+
   # Publicly accessible filename for the resulting report
   attr_reader :filename
 
+  # A custom subclass that never calls setup() still gets a usable filename, e.g.
+  # 'foo_bar.report' for a report named 'foo_bar'
   def initialize(handle:)
     @handle = handle
-
-    # Safe default filename in case user's custom subclass forgets to call 
-    # setup() with a default filename.
-    # If the report is named 'foo_bar' in project configuration, the 
-    # fallback filename is 'foo_bar.report'
     @filename = "#{handle}.report"
   end
 
@@ -30,13 +38,13 @@ class TestsReporter
     @filename = update_filename( default_filename )
   end
 
-  # Write report contents to file
+  # Renders the report's sections in order, then writes the whole report in one call
   def write(name:, filepath:, results:, duration_s:nil)
-    File.open( filepath, 'w' ) do |f|
-      header( stream: f, name: name, results: results, duration_s: duration_s )
-      body( stream: f, name: name, results: results, duration_s: duration_s )
-      footer( stream: f, name: name, results: results, duration_s: duration_s )
+    buffer = StringIO.new
+    [:header, :body, :footer].each do |section|
+      public_send( section, stream: buffer, name: name, results: results, duration_s: duration_s )
     end
+    @file_wrapper.write( filepath, buffer.string )
   end
 
   def header(stream:, name:, results:, duration_s:)
@@ -55,8 +63,8 @@ class TestsReporter
 
   private
 
+  # The configured filename, or the subclass's default
   def update_filename(default_filename)
-    # Fetch configured filename if it exists, otherwise return default filename
     filename, _ = @config_walkinator.fetch_value( :filename, hash:@config, default:default_filename )
     return filename
   end
@@ -67,4 +75,15 @@ class TestsReporter
     return result
   end
 
-end
+  # Escapes text for an XML attribute or element body. Every reporter receives the same
+  # results, so this returns a new string and never alters a name or message in place.
+  def xml_escape(str)
+    str.to_s.gsub( /[&<>"']/, '&' => '&amp;', '<' => '&lt;', '>' => '&gt;', '"' => '&quot;', "'" => '&apos;' )
+  end
+
+  # Escapes text for HTML. Like xml_escape, it returns a new string.
+  def html_escape(str)
+    CGI.escapeHTML( str.to_s )
+  end
+
+end