ceedling 0.9.0 → 1.0.1

data/lib/ceedling/generator_test_results.rb

@@ -0,0 +1,270 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'rubygems'
+require 'rake' # for .ext()
+require 'ceedling/constants'
+require 'ceedling/exceptions'
+
+
+##
+## Sample Unity Test Executable Output
+## ===================================
+##
+## - Output is line-oriented. Anything outside the recognized lines is assumed to be from `printf()`
+##   or equivalent calls and collected for presentation as a collection of $stdout lines.
+## - Multiline output (i.e. failure messages) can be achieved by "encoding" newlines as literal 
+##   "\n"s (slash-n). `extract_line_elements()` handles converting newline markers into real newlines.
+## - :PASS has no trailing message unless Unity's test case execution duration feature is enabled.
+##   If enabled, a numeric value with 'ms' as a units signifier trails, ":PASS 1.2 ms".
+## - :IGNORE optionally can include a trailing message.
+## - :FAIL has a trailing message that relays an assertion failure or crash condition.
+## - The statistics line always has the same format with only the count values varying.
+## - If there are no failed test cases, the final line is 'OK'. Otherwise, it is 'FAIL'.
+##
+## $stdout:
+## -----------------------------------------------------------------------------------------------------
+## TestUsartModel.c:24:testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting:PASS
+## TestUsartModel.c:34:testIgnore:IGNORE
+## TestUsartModel.c:39:testFail:FAIL: Expected 2 Was 3
+## TestUsartModel.c:49:testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately:PASS
+## TestUsartModel.c:55:testShouldReturnErrorMessageUponInvalidTemperatureValue:PASS
+## TestUsartModel.c:61:testShouldReturnWakeupMessage:PASS
+##
+## -----------------------
+## 6 Tests 1 Failures 1 Ignored
+## FAIL
+
+##
+## Sample Test Results Output File (YAML)
+## ======================================
+## The following corresponds to the test executable output above.
+##
+## TestUsartModel.fail:
+## ---
+## :source:
+##   :file: test/TestUsartModel.c
+##   :dirname: test
+##   :basename: TestUsartModel.c
+## :successes:
+## - :test: testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting
+##   :line: 24
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately
+##   :line: 49
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testShouldReturnErrorMessageUponInvalidTemperatureValue
+##   :line: 55
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testShouldReturnWakeupMessage
+##   :line: 61
+##   :message: ''
+##   :unity_test_time: 0
+## :failures:
+## - :test: testFail
+##   :line: 39
+##   :message: Expected 2 Was 3
+##   :unity_test_time: 0
+## :ignores:
+## - :test: testIgnore
+##   :line: 34
+##   :message: ''
+##   :unity_test_time: 0
+## :counts:
+##   :total: 6
+##   :passed: 4
+##   :failed: 1
+##   :ignored: 1
+## :stdout: []
+## :time: 0.006512000225484371
+
+class GeneratorTestResults
+
+  constructor :configurator, :generator_test_results_sanity_checker, :yaml_wrapper
+
+  def setup()
+    # Aliases
+    @sanity_checker = @generator_test_results_sanity_checker
+  end
+
+  def process_and_write_results(executable, unity_shell_result, results_file, test_file)
+    output_file = results_file
+
+    results = get_results_structure
+
+    results[:source][:dirname] = File.dirname(test_file)
+    results[:source][:basename] = File.basename(test_file)
+    results[:source][:file] = test_file
+    results[:time] = unity_shell_result[:time] unless unity_shell_result[:time].nil?
+
+    # Process test statistics
+    if (unity_shell_result[:output] =~ PATTERNS::TEST_STDOUT_STATISTICS)
+      results[:counts][:total] =   $1.to_i
+      results[:counts][:failed] =  $2.to_i
+      results[:counts][:ignored] = $3.to_i
+      results[:counts][:passed] = (results[:counts][:total] - results[:counts][:failed] - results[:counts][:ignored])
+    else
+      raise CeedlingException.new( "Could not parse output for `#{executable}`: \"#{unity_shell_result[:output]}\"" ) 
+    end
+
+    # Remove test statistics lines
+    output_string = unity_shell_result[:output].sub( PATTERNS::TEST_STDOUT_STATISTICS, '' )
+
+    # Process test executable results line-by-line
+    output_string.lines do |line|
+      # Process Unity test executable output
+      case line.chomp
+      when /(:IGNORE)/
+        elements = extract_line_elements( executable, line, results[:source][:file] )
+        results[:ignores] << elements[0] 
+        results[:stdout] << elements[1] if (!elements[1].nil?)
+
+      when /(:PASS$)/
+        elements = extract_line_elements( executable, line, results[:source][:file] )
+        results[:successes] << elements[0] 
+        results[:stdout] << elements[1] if (!elements[1].nil?)
+
+      when /(:PASS \(.* ms\)$)/
+        elements = extract_line_elements( executable, line, results[:source][:file] )
+        results[:successes] << elements[0] 
+        results[:stdout] << elements[1] if (!elements[1].nil?)
+
+      when /(:FAIL)/
+        elements = extract_line_elements( executable, line, results[:source][:file] )
+        results[:failures] << elements[0]
+        results[:stdout] << elements[1] if (!elements[1].nil?)
+
+      # Collect up all other output
+      else
+        results[:stdout] << line.chomp # Ignores blank lines
+      end
+    end
+
+    @sanity_checker.verify( results, unity_shell_result[:exit_code] )
+
+    output_file = results_file.ext( @configurator.extension_testfail ) if (results[:counts][:failed] > 0)
+
+    @yaml_wrapper.dump(output_file, results)
+
+    return { :result_file => output_file, :result => results }
+  end
+
+  # Filter list of test cases:
+  #  --test_case
+  #  --exclude_test_case
+  #
+  # @return Array - list of the test_case hashses {:test, :line_number}
+  def filter_test_cases(test_cases)
+    _test_cases = test_cases.clone 
+
+    # Filter tests which contain test_case_name passed by `--test_case` argument
+    if !@configurator.include_test_case.empty?
+      _test_cases.delete_if { |i| !(i[:test] =~ /#{@configurator.include_test_case}/) }
+    end
+
+    # Filter tests which contain test_case_name passed by `--exclude_test_case` argument
+    if !@configurator.exclude_test_case.empty?
+      _test_cases.delete_if { |i| i[:test] =~ /#{@configurator.exclude_test_case}/ }
+    end
+
+    return _test_cases
+  end
+
+  def create_crash_failure(source, shell_result, test_cases)
+    count = test_cases.size()
+
+    output = []
+    test_cases.each do |test_case|
+      output << "#{source}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test executable crashed"
+    end
+
+    shell_result[:output] = 
+      regenerate_test_executable_stdout(
+        total:   count,
+        failed:  count,
+        ignored: 0,
+        output:  output
+      )
+
+    shell_result[:exit_code] = count
+
+    return shell_result
+  end
+
+  # Fill out a template to mimic Unity's test executable output
+  def regenerate_test_executable_stdout(total:, failed:, ignored:, output:[])
+    values = {
+      :total => total,
+      :failed => failed,
+      :ignored => ignored,
+      :output => output.map {|line| line.strip()}.join("\n"),
+      :result => (failed > 0) ? 'FAIL' : 'OK'
+    }
+
+    return UNITY_TEST_RESULTS_TEMPLATE % values
+  end
+
+  ### Private ### 
+  
+  private
+
+  def get_results_structure
+    return {
+      :source    => {:file => '', :dirname => '', :basename => '' },
+      :successes => [],
+      :failures  => [],
+      :ignores   => [],
+      :counts    => {:total => 0, :passed => 0, :failed => 0, :ignored  => 0},
+      :stdout    => [],
+      :time      => 0.0
+      }
+  end
+
+  def extract_line_elements(executable, line, filename)
+    # Handle anything preceding filename in line as extra output to be collected
+    stdout = nil
+    stdout_regex = /(.+)#{Regexp.escape(filename)}:[0-9]+:(PASS|IGNORE|FAIL).+/i
+    unity_test_time = 0 
+
+    if (line =~ stdout_regex)
+      stdout = $1.clone
+      line.sub!(/#{Regexp.escape(stdout)}/, '')
+    end
+
+    # Collect up test results minus any extra output
+    elements = (line.strip.split(':'))[1..-1]
+
+    # Find timestamp if available
+    if (elements[-1] =~ / \((\d*(?:\.\d*)?) ms\)/)
+      unity_test_time = $1.to_f / 1000
+      elements[-1].sub!(/ \((\d*(?:\.\d*)?) ms\)/, '')
+    end
+
+    if elements[3..-1]
+      message = (elements[3..-1].join(':')).strip
+    else
+      message = nil
+    end
+
+    components = {
+      :test => elements[1],
+      :line => elements[0].to_i,
+      # Decode any multline strings
+      :message => message.nil? ? nil : message.gsub( NEWLINE_TOKEN, "\n" ),
+      :unity_test_time => unity_test_time
+    }
+
+    return components, stdout if elements.size >= 3
+    
+    # Fall through failure case
+    raise CeedlingException.new( "Could not parse results output line \"line\" for `#{executable}`" )
+  end
+
+end

data/lib/ceedling/generator_test_results_backtrace.rb

@@ -0,0 +1,215 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+class GeneratorTestResultsBacktrace
+
+  constructor :configurator, :tool_executor, :generator_test_results
+
+  def setup()
+    @RESULTS_COLLECTOR = Struct.new( :passed, :failed, :ignored, :output, keyword_init:true )
+  end
+
+  def do_simple(filename, executable, shell_result, test_cases)
+    # Clean stats tracker
+    test_case_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
+
+    # Reset time
+    shell_result[:time] = 0
+
+    # Iterate on test cases
+    test_cases.each do |test_case|
+      # Build the test fixture to run with our test case of interest
+      command = @tool_executor.build_command_line(
+        @configurator.tools_test_fixture_simple_backtrace, [],
+        executable,
+        test_case[:test]
+      )
+      # Things are gonna go boom, so ignore booms to get output
+      command[:options][:boom] = false
+
+      crash_result = @tool_executor.exec( command )
+
+      # Sum execution time for each test case
+      # Note: Running tests serpatately increases total execution time)
+      shell_result[:time] += crash_result[:time].to_f()
+
+      # Process single test case stats
+      case crash_result[:output]
+      # Success test case
+      when /(^#{filename}.+:PASS\s*$)/
+        test_case_results[:passed]  += 1
+        test_output = $1 # Grab regex match
+
+      # Ignored test case
+      when /(^#{filename}.+:IGNORE\s*$)/
+        test_case_results[:ignored] += 1
+        test_output = $1 # Grab regex match
+
+      when /(^#{filename}.+:FAIL(:.+)?\s*$)/
+        test_case_results[:failed]  += 1
+        test_output = $1 # Grab regex match
+
+      else # Crash failure case
+        test_case_results[:failed]  += 1
+        test_output = "#{filename}}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed"
+      end
+
+      # Collect up real and stand-in test results output
+      test_case_results[:output] << test_output
+    end
+
+    # Reset shell result exit code and output
+    shell_result[:exit_code] = test_case_results[:failed]
+    shell_result[:output] =
+      @generator_test_results.regenerate_test_executable_stdout(
+        total:   test_cases.size(),
+        ignored: test_case_results[:ignored],
+        failed:  test_case_results[:failed],
+        output:  test_case_results[:output]
+      )
+
+    return shell_result
+  end
+
+  def do_gdb(filename, executable, shell_result, test_cases)
+    gdb_script_filepath = File.join( @configurator.project_build_tests_root, BACKTRACE_GDB_SCRIPT_FILE )
+
+    # Clean stats tracker
+    test_case_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
+
+    # Reset time
+    shell_result[:time] = 0
+
+    # Iterate on test cases
+    test_cases.each do |test_case|
+      # Build the test fixture to run with our test case of interest
+      command = @tool_executor.build_command_line(
+        @configurator.tools_test_backtrace_gdb, [],
+        gdb_script_filepath,
+        executable,
+        test_case[:test]
+      )
+      # Things are gonna go boom, so ignore booms to get output
+      command[:options][:boom] = false
+
+      crash_result = @tool_executor.exec( command )
+
+      # Sum execution time for each test case
+      # Note: Running tests serpatately increases total execution time)
+      shell_result[:time] += crash_result[:time].to_f()
+
+      test_output = ''
+
+      # Process single test case stats
+      case crash_result[:output]
+      # Success test case
+      when /(^#{filename}.+:PASS\s*$)/
+        test_case_results[:passed]  += 1
+        test_output = $1 # Grab regex match
+
+      # Ignored test case
+      when /(^#{filename}.+:IGNORE\s*$)/
+        test_case_results[:ignored] += 1
+        test_output = $1 # Grab regex match
+
+      when /(^#{filename}.+:FAIL(:.+)?\s*$)/
+        test_case_results[:failed]  += 1
+        test_output = $1 # Grab regex match
+
+      else # Crash failure case
+        test_case_results[:failed]  += 1
+
+        # Collect file_name and line in which crash occurred
+        matched = crash_result[:output].match( /#{test_case[:test]}\s*\(\)\sat.+#{filename}:(\d+)\n/ )
+
+        # If we found an error report line containing `test_case() at filename.c:###` in `gdb` output
+        if matched
+          # Line number
+          line_number = matched[1]
+
+          # Filter the `gdb` $stdout report to find most important lines of text
+          crash_report = filter_gdb_test_report( crash_result[:output], test_case[:test], filename )
+
+          # Unity’s test executable output is line oriented.
+          # Multi-line output is not possible (it looks like random `printf()` statements to the results parser)
+          # "Encode" newlines in multiline string to be handled by the test results parser.
+          test_output = crash_report.gsub( "\n", NEWLINE_TOKEN )
+
+          test_output = "#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed >> #{test_output}"
+
+        # Otherwise communicate that `gdb` failed to produce a usable report
+        else
+          test_output = "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed (no usable `gdb` report)"
+        end
+      end
+
+      test_case_results[:output] << test_output
+    end
+
+    # Reset shell result exit code and output
+    shell_result[:exit_code] = test_case_results[:failed]
+    shell_result[:output] =
+      @generator_test_results.regenerate_test_executable_stdout(
+        total:   test_cases.size(),
+        ignored: test_case_results[:ignored],
+        failed:  test_case_results[:failed],
+        output:  test_case_results[:output]
+      )
+
+    return shell_result
+  end
+
+  ### Private ###
+  private
+
+  def filter_gdb_test_report(report, test_case, filename)
+    #       Sample `gdb` backtrace output
+    #       =============================
+    #       [Thread debugging using libthread_db enabled]
+    #       Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
+    #
+    # [1] > Program received signal SIGSEGV, Segmentation fault.
+    #       0x0000555999f661fb in testCrash () at test/TestUsartModel.c:40
+    # [2] > 40    uint32_t i = *nullptr;
+    #       #0  0x0000555999f661fb in testCrash () at test/TestUsartModel.c:40
+    #       #1  0x0000555999f674de in run_test (func=0x555999f661e7 <testCrash>, name=0x555999f6b2e0 "testCrash", line_num=37) at build/test/runners/TestUsartModel_runner.c:76
+    #       #2  0x0000555999f6766b in main (argc=3, argv=0x7fff917e2c98) at build/test/runners/TestUsartModel_runner.c:117
+
+    lines = report.split( "\n" )
+
+    # Safe defaults to extract entire report
+    report_start_index = 0                # [1]
+    report_end_index   = (lines.size()-1) # [2]
+
+    # Find line preceding last `<test_case> () at <filename>`, [2];
+    # it is the offending line of code.
+    # We don't need the rest of the call trace -- it's just from the runner 
+    # up to the crashed test case.
+    lines.each_with_index do |line, index|
+      if line =~ /#{test_case}.+at.+#{filename}/
+        report_end_index = (index - 1) unless (index == 0)
+      end
+    end
+
+    # Work up from [2] to find top line of the containing text block, [1].
+    # Go until we find a blank line; then increment index back down a line.
+    # This lops off any unneeded preamble.
+    report_end_index.downto(0).to_a().each do |index|
+      if lines[index].empty?
+        # Look for a blank line and adjust index to next line of text
+        report_start_index = (index + 1)
+        break
+      end
+    end
+
+    length = (report_end_index - report_start_index) + 1
+
+    # Reconstitute the report from the extracted lines
+    return lines[report_start_index, length].join( "\n" )
+  end
+
+end

data/lib/ceedling/generator_test_results_sanity_checker.rb

@@ -0,0 +1,70 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'rubygems'
+require 'rake' # for ext() method
+require 'ceedling/constants'
+require 'ceedling/exceptions'
+
+
+class GeneratorTestResultsSanityChecker
+
+  constructor :configurator, :loginator
+  
+  def verify(results, unity_exit_code)
+  
+    # do no sanity checking if it's disabled
+    return if (@configurator.sanity_checks == TestResultsSanityChecks::NONE)
+    raise CeedlingException.new( "Test results nil or empty" ) if results.nil? || results.empty?
+
+    ceedling_ignores_count   = results[:ignores].size 
+    ceedling_failures_count  = results[:failures].size
+    ceedling_tests_summation = (ceedling_ignores_count + ceedling_failures_count + results[:successes].size)
+
+    # Exit code handling is not a sanity check that can always be performed because 
+    # command line simulators may or may not pass through Unity's exit code
+    if (@configurator.sanity_checks >= TestResultsSanityChecks::THOROUGH)
+      # many platforms limit exit codes to a maximum of 255
+      if ((ceedling_failures_count != unity_exit_code) and (unity_exit_code < 255))
+        sanity_check_warning(results[:source][:file], "Unity's exit code (#{unity_exit_code}) does not match Ceedling's summation of failed test cases (#{ceedling_failures_count}).")
+      end
+      
+      if ((ceedling_failures_count < 255) and (unity_exit_code == 255))
+        sanity_check_warning(results[:source][:file], "Ceedling's summation of failed test cases (#{ceedling_failures_count}) is less than Unity's exit code (255 or more).")
+      end
+    end
+    
+    if (ceedling_ignores_count != results[:counts][:ignored])
+      sanity_check_warning(results[:source][:file], "Unity's final ignore count (#{results[:counts][:ignored]}) does not match Ceedling's summation of ignored test cases (#{ceedling_ignores_count}).")
+    end
+    
+    if (ceedling_failures_count != results[:counts][:failed])
+      sanity_check_warning(results[:source][:file], "Unity's final fail count (#{results[:counts][:failed]}) does not match Ceedling's summation of failed test cases (#{ceedling_failures_count}).")
+    end
+
+    if (ceedling_tests_summation != results[:counts][:total])
+      sanity_check_warning(results[:source][:file], "Unity's final test count (#{results[:counts][:total]}) does not match Ceedling's summation of all test cases (#{ceedling_tests_summation}).")
+    end
+    
+  end
+
+  private
+  
+  def sanity_check_warning(file, message)
+    unless defined?(CEEDLING_IGNORE_SANITY_CHECK)
+      notice = "Internal sanity check for test fixture '#{file.ext(@configurator.extension_executable)}' finds that #{message}\n" +
+               "  Possible causes:\n" +
+               "    1. Your test + source dereferenced a null pointer.\n" +
+               "    2. Your test + source indexed past the end of a buffer.\n" +
+               "    3. Your test + source committed a memory access violation.\n" +
+               "    4. Your test fixture produced an exit code of 0 despite execution ending prematurely.\n" +
+               "  Sanity check failures of test results are usually a symptom of interrupted test execution.\n\n"      
+      raise CeedlingException.new( notice )
+    end
+  end
+
+end

data/lib/ceedling/generator_test_runner.rb

@@ -0,0 +1,84 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'generate_test_runner' # Unity's test runner generator
+
+class GeneratorTestRunner
+
+  attr_accessor :test_cases
+
+  #
+  # This class is not within any DIY context.
+  # It is instantiated on demand for each test file processed in a build.
+  #
+
+  def initialize(config:, test_file_contents:, preprocessed_file_contents:nil)
+    @unity_runner_generator = UnityTestRunnerGenerator.new( config )
+    
+    # Reduced information set
+    @test_cases = []
+
+    # Full information set used for runner generation
+    @test_cases_internal = []
+
+    parse_test_file( test_file_contents, preprocessed_file_contents )
+  end
+
+  def generate(module_name:, runner_filepath:, mock_list:, test_file_includes:, header_extension:)
+    # Actually build the test runner using Unity's test runner generator.
+    @unity_runner_generator.generate(
+      module_name,
+      runner_filepath,
+      @test_cases_internal,
+      mock_list.map{ |mock| mock + header_extension },
+      test_file_includes.map{|f| File.basename(f,'.*') + header_extension}
+    )
+  end
+
+  ### Private ###
+
+  private
+
+  def parse_test_file(test_file_contents, preprocessed_file_contents)
+    # If there's a preprocessed file, align test case line numbers with original file contents
+    if (!preprocessed_file_contents.nil?)
+      # Save the test case structure to be used in generation
+      @test_cases_internal = @unity_runner_generator.find_tests( preprocessed_file_contents )
+      
+      # Configure the runner generator around `setUp()` and `tearDown()`
+      @unity_runner_generator.find_setup_and_teardown( preprocessed_file_contents )
+
+      # Modify line numbers to match the original, non-preprocessed file
+      source_lines = test_file_contents.split("\n")
+      source_index = 0;
+      @test_cases_internal.size.times do |i|
+        source_lines[source_index..-1].each_with_index do |line, index|
+          if (line =~ /#{@test_cases_internal[i][:test]}/)
+            source_index += index
+            @test_cases_internal[i][:line_number] = source_index + 1
+            break
+          end
+        end
+      end
+
+    # Just look for the test cases within the original test file
+    else
+      # Save the test case structure to be used in generation
+      @test_cases_internal = @unity_runner_generator.find_tests( test_file_contents )
+
+      # Configure the runner generator around `setUp()` and `tearDown()`
+      @unity_runner_generator.find_setup_and_teardown( test_file_contents )
+    end
+
+    # Unity's `find_tests()` produces an array of hashes with the following keys...
+    # { test:, args:, call:, params:, line_number: }
+
+    # For external use of test case names and line numbers, keep only those pieces of info
+    @test_cases = @test_cases_internal.map {|hash| hash.slice( :test, :line_number )}
+  end
+
+end