ceedling 0.9.0 → 1.0.1

data/plugins/report_tests_pretty_stdout/README.md

@@ -0,0 +1,54 @@
+# Ceedling Plugin: Pretty Test Suite Console Report
+
+Prints to the console ($stdout) simple, readable test suite results.
+
+# Plugin Overview
+
+This plugin is intended to be the default option for formatting a test suite's
+results when displayed at the console. It collects raw test results from the 
+individual test executables of your test suite and presents them in a more 
+readable summary form.
+
+# Setup
+
+Enable the plugin in your project.yml by adding `report_tests_pretty_stdout` to 
+the list of enabled plugins instead of any other `report_tests_*_stdout` 
+plugin.
+
+``` YAML
+:plugins:
+  :enabled:
+    - report_tests_pretty_stdout
+```
+
+# Configuration
+
+No additional configuration is needed once the plugin is enabled.
+
+# Example Output
+
+```sh
+ > ceedling test:Model
+```
+
+```
+-------------------
+FAILED TEST SUMMARY
+-------------------
+[test/TestModel.c]
+  Test: testInitShouldCallSchedulerAndTemperatureFilterInit
+  At line (21): "Function TaskScheduler_Init() called more times than expected."
+
+--------------------
+OVERALL TEST SUMMARY
+--------------------
+TESTED:  1
+PASSED:  0
+FAILED:  1
+IGNORED: 0
+
+---------------------
+BUILD FAILURE SUMMARY
+---------------------
+Unit test failures.
+```

data/plugins/report_tests_pretty_stdout/assets/template.erb

@@ -0,0 +1,63 @@
+% ignored        = hash[:results][:counts][:ignored]
+% failed         = hash[:results][:counts][:failed]
+% stdout_count   = hash[:results][:counts][:stdout]
+% header_prepend = ((hash[:header].length > 0) ? "#{hash[:header]}: " : '')
+% banner_width   = 25 + header_prepend.length # widest message
+
+% if (stdout_count > 0)
+<%=@ceedling[:plugin_reportinator].generate_banner(header_prepend + 'TEST OUTPUT')%>
+%   hash[:results][:stdout].each do |string|
+[<%=string[:source][:file]%>]
+%     string[:collection].each do |item|
+  - "<%=item%>"
+%     end
+
+%   end
+% end
+% if (ignored > 0)
+<%=@ceedling[:plugin_reportinator].generate_banner(header_prepend + 'IGNORED TEST SUMMARY')%>
+%   hash[:results][:ignores].each do |ignore|
+[<%=ignore[:source][:file]%>]
+%     ignore[:collection].each do |item|
+  Test: <%=item[:test]%>
+%       if (not item[:message].empty?)
+  At line (<%=item[:line]%>): "<%=item[:message]%>"
+%       else
+  At line (<%=item[:line]%>)
+%       end
+
+%     end
+%   end
+% end
+% if (failed > 0)
+<%=@ceedling[:plugin_reportinator].generate_banner(header_prepend + 'FAILED TEST SUMMARY')%>
+%   hash[:results][:failures].each do |failure|
+[<%=failure[:source][:file]%>]
+%     failure[:collection].each do |item|
+%       header = "At line (#{item[:line]}):"
+  Test: <%=item[:test]%>
+%       if (not item[:message].empty?)
+%         # Cause multline error messages to wrap with indentation aligned with line header
+%         msg = item[:message].split("\n").enum_for(:each_with_index).map{|line,i| ((i >= 1) ? (' ' * (header.size + 3)) : '') + line}.join("\n")
+  <%=header%> "<%=msg%>"
+%       else
+  <%=header%>
+%       end
+
+%     end
+%   end
+% end
+% total_string = hash[:results][:counts][:total].to_s
+% format_string = "%#{total_string.length}i"
+% decorator = (failed > 0) ? LogLabels::FAIL : LogLabels::PASS
+<%=@ceedling[:plugin_reportinator].generate_banner(header_prepend + @loginator.decorate('OVERALL TEST SUMMARY', decorator))%>
+% if (hash[:results][:counts][:total] > 0)
+TESTED:  <%=hash[:results][:counts][:total].to_s%>
+PASSED:  <%=sprintf(format_string, hash[:results][:counts][:passed])%>
+FAILED:  <%=sprintf(format_string, failed)%>
+IGNORED: <%=sprintf(format_string, ignored)%>
+% else
+
+No tests executed.
+% end
+

data/plugins/report_tests_pretty_stdout/config/report_tests_pretty_stdout.yml

@@ -0,0 +1,11 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+---
+:plugins:
+  # tell Ceedling we got results display taken care of
+  :display_raw_test_results: FALSE

data/plugins/report_tests_pretty_stdout/lib/report_tests_pretty_stdout.rb

@@ -0,0 +1,73 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/plugin'
+require 'ceedling/defaults'
+
+class ReportTestsPrettyStdout < Plugin
+
+  # `Plugin` setup()
+  def setup
+    @result_list = []
+    @mutex = Mutex.new
+    
+    # Fetch the test results template for this plugin
+    template = @ceedling[:file_wrapper].read( File.join( @plugin_root_path, 'assets/template.erb' ) )
+
+    # Set the report template
+    @ceedling[:plugin_reportinator].register_test_results_template( template )
+  end
+  
+  # `Plugin` build step hook -- collect result file paths after each test fixture execution
+  def post_test_fixture_execute(arg_hash)
+    # Thread-safe manipulation since test fixtures can be run in child processes
+    # spawned within multiple test execution threads.
+    @mutex.synchronize do
+      @result_list << arg_hash[:result_file]
+    end
+  end
+  
+  # `Plugin` build step hook -- render a report immediately upon build completion (that invoked tests)
+  def post_build()
+    # Ensure a test task was invoked as part of the build
+    return if not (@ceedling[:task_invoker].test_invoked?)
+
+    results = @ceedling[:plugin_reportinator].assemble_test_results( @result_list )
+    hash = {
+      :header => '',
+      :results => results
+    }
+
+    @ceedling[:plugin_reportinator].run_test_results_report(hash) do
+      message = ''
+      message = 'Unit test failures.' if (results[:counts][:failed] > 0)
+      message
+    end
+  end
+
+  # `Plugin` build step hook -- render a test results report on demand using results from a previous build
+  def summary()
+    # Build up the list of passing results from all tests
+    result_list = @ceedling[:file_path_utils].form_pass_results_filelist(
+      PROJECT_TEST_RESULTS_PATH,
+      COLLECTION_ALL_TESTS
+    )
+
+    hash = {
+      :header => '',
+      # Collect all existing test results (success or failing) in the filesystem,
+      # limited to our test collection
+      :results => @ceedling[:plugin_reportinator].assemble_test_results(
+        result_list,
+        {:boom => false}
+      )
+    }
+
+    @ceedling[:plugin_reportinator].run_test_results_report( hash )
+  end
+
+end

data/plugins/report_tests_raw_output_log/README.md

@@ -0,0 +1,50 @@
+# Ceedling Plugin: Raw Test Output Logs
+
+Capture extra console output — typically `printf()`-style statements — from 
+test cases to log files.
+
+# Plugin Overview
+
+This plugin gathers and filters console output from test executables into log 
+files. Though not required, it is usually used in addition to the 
+`report_tests_*_stdout` plugins that gather and format test results for display
+at the console.
+
+Debugging in unit tested code is often accomplished with simple `printf()`-
+style calls to dump information to the console. This plugin's log files can be 
+helpful in supporting debugging efforts or quality validation.
+
+## Test executable output
+
+Ceedling and Unity cooperate to extract console statements from test executable
+runs. Unity-based test executables print test case pass/fail status messages
+and test case accounting to the console ($stdout).
+
+Ceedling and various reporting plugins gather all this, including unrecognized 
+output, to format it and present summaries at the console.
+
+This plugin captures the unrecognized output to log files.
+
+## Log files
+
+Log files are only created if test executables produce console output apart from
+expected Unity test results as described above. Log files are named for the
+respective test executables.
+
+Builds are differentiated by build context — `test`, `release`, or
+plugin-modified build (e.g. `gcov`). Log files are written to `<build
+root>/artifacts/<context>/<test file>.raw.log`.
+
+# Setup
+
+Enable the plugin in your Ceedling project:
+
+``` YAML
+:plugins:
+  :enabled:
+    - report_tests_raw_output_log
+```
+
+# Configuration
+
+No additional configuration is needed once the plugin is enabled.

data/plugins/report_tests_raw_output_log/lib/report_tests_raw_output_log.rb

@@ -0,0 +1,129 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/plugin'
+require 'ceedling/constants'
+
+class ReportTestsRawOutputLog < Plugin
+  # `Plugin` setup()
+  def setup
+   # @raw_output hash with default values
+    @raw_output = {}
+
+    # Ceedling can run with multiple threads, provide a lock to use around @raw_output
+    @mutex = Mutex.new()
+
+    # Convenient instance variable references
+    @file_wrapper = @ceedling[:file_wrapper]
+    @loginator = @ceedling[:loginator]
+    @reportinator = @ceedling[:reportinator]
+  end
+
+  # `Plugin` build step hook
+  def post_test_fixture_execute(arg_hash)
+    output = extract_output( arg_hash[:shell_result][:output] )
+
+    # Bail out early
+    return if output.empty?
+    
+    # After test fixture execution, parse output, store any raw console statements
+    @mutex.synchronize do
+      process_output(
+        arg_hash[:context],
+        arg_hash[:test_name],
+        output,
+        @raw_output
+      )
+    end
+  end
+
+  # `Plugin` build step hook
+  def post_build()
+    # Write collected raw output to log(s)
+    write_logs( @raw_output )
+  end
+
+  # `Plugin` build step hook
+  def post_error()
+    # Write collected raw output to log(s)
+    write_logs( @raw_output )
+  end
+
+  ### Private ###
+
+  private
+
+  # Pick apart test executable console output to find any lines not specific to a test case
+  def extract_output(raw_output)
+    output = []
+
+    raw_output.each_line do |line|
+      # Skip blank lines
+      next if line =~ /^\s*\n$/
+
+      # Skip test case reporting lines
+      next if line =~ /^.+:\d+:.+:(IGNORE|PASS|FAIL)/
+
+      # Return early if we get to test results summary footer
+      return output if line =~/^-+\n$/
+
+      # Capture all other console output from the test runner, including `printf()`-style debugging statements
+      output << line
+    end
+
+    return output
+  end
+
+  # Store raw output messages to hash in thread-safe manner
+  def process_output(context, test, output, hash)
+    # Store warning message
+    hash[context] = {} if hash[context].nil?
+    hash[context][test] = output
+  end
+
+  def write_logs(hash)
+    msg = @reportinator.generate_heading( "Running Raw Tests Output Report" )
+    @loginator.log( msg )
+
+    empty = false
+
+    @mutex.synchronize { empty = hash.empty? }
+
+    if empty
+      @loginator.log( "Tests produced no extra console output.\n" )
+      return
+    end
+
+    @mutex.synchronize do
+      hash.each do |context, tests|
+        tests.each do |test, output|
+          log_filepath = form_log_filepath( context, test )
+
+          msg = @reportinator.generate_progress( "Generating artifact #{log_filepath}" )
+          @loginator.log( msg )
+
+          File.open( log_filepath, 'w' ) do |f|
+            output.each { |line| f << line }
+          end
+        end
+      end
+    end
+
+    # White space at command line after progress messages
+    @loginator.log( '' )
+  end
+
+  def form_log_filepath(context, test)
+    path = File.join( PROJECT_BUILD_ARTIFACTS_ROOT, context.to_s )
+    filepath = File.join(path, test + '.raw.log')
+
+    # Ensure containing artifact directory exists
+    @file_wrapper.mkdir( path )
+
+    return filepath
+  end
+end

data/plugins/report_tests_teamcity_stdout/README.md

@@ -0,0 +1,94 @@
+# Ceedling Plugin: TeamCity Test Suite Console Report
+
+Prints to the console ($stdout) test suite build events and results in a format understood by the CI product TeamCity.
+
+# Plugin Overview
+
+This plugin is intended to be used within [TeamCity] Continuous Integration
+(CI) builds. It processes Ceedling test suites and executable output into
+TeamCity [service messages][service-messages]. Service messages are a specially
+formatted type of log output that a TeamCity build server picks out of build
+output to collect progress and metrics of various sorts.
+
+Typically, this plugin is used only in CI builds. Its output is unhelpful in
+development builds locally. See the [Configuration](#configuration) section for
+options on enabling the build in CI but disabling it locally.
+
+[TeamCity]: https://www.jetbrains.com/teamcity/
+[service-messages]:
+https://www.jetbrains.com/help/teamcity/service-messages.html
+
+# Setup
+
+Enable the plugin in your Ceedling project file by adding 
+`report_tests_teamcity_stdout`.
+
+``` YAML
+:plugins:
+  :enabled:
+    - report_tests_teamcity_stdout
+```
+
+# Configuration
+
+All the `report_tests_*_stdout` plugins may be enabled in various combinations.
+But, some combinations may not make a great deal of sense. The TeamCity
+plugin “plays nice” with all the others but is generally most appropriate 
+running in a CI build on a TeamCity server. Its output will clutter and obscure
+console logging at a local development environment command line.
+
+You may enable the TeamCity plugin (above) but disable its operation using the
+following.
+
+```YAML
+:teamcity:
+  :build: FALSE
+```
+
+This may seem silly, right? Why enable the plugin and then disable it,
+cancelling it out? The answer has to do with _where_ you use the second YAML
+blurb configuration setting.
+
+Ceedling provides Mixins for applying configurations settings on top of your
+base project configuraiton file. 
+See the [Mixins documentation][ceedling-mixins] for full details.
+
+[ceedling-mixins]: ../docs/CeedlingPacket.md#base-project-configuration-file-mixins-section-entries
+
+As an example, you might enable the plugin in the main project file that is
+committed to your repository while disabling the plugin in your local user
+project file that is ignored by your repository. In this way, the plugin would
+run on a TeamCity build server but not in your local development environment.
+
+# Example Output
+
+TeamCity's convention for identifying tests uses the naming convention of the
+underlying Java language in which TeamCity is written,
+`package_or_namespace.ClassName.TestName`.
+
+This plugin maps Ceedling conventions to TeamCity test service messages as
+`context.TestFilepath.TestCaseName`.
+
+* `context` Your build's context defaults to `test`. Certain other test build 
+  plugins (e.g. GCov) provide a different context (`gcov`) for test builds, 
+  generally named after themselves.
+* `TestFilepath` This identifier is the relative filepath of the relevant test
+  file without a file extension (e.g. no `.c`).
+* `TestCaseName` This identifier is a test case function name within a Ceedling test file.
+
+```sh
+ > ceedling test:UsartModel
+```
+
+```
+##teamcity[testSuiteStarted name='TestUsartModel' flowId='15']
+##teamcity[testStarted name='test.test/TestUsartModel.testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting' flowId='15']
+##teamcity[testFinished name='test.test/TestUsartModel.testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting' duration='81' flowId='15']
+##teamcity[testStarted name='test.test/TestUsartModel.testShouldReturnErrorMessageUponInvalidTemperatureValue' flowId='15']
+##teamcity[testFinished name='test.test/TestUsartModel.testShouldReturnErrorMessageUponInvalidTemperatureValue' duration='81' flowId='15']
+##teamcity[testStarted name='test.test/TestUsartModel.testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately' flowId='15']
+##teamcity[testFailed name='test.test/TestUsartModel.testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately' message='Function TemperatureFilter_GetTemperatureInCelcius() called more times than expected.' details='File: test/TestUsartModel.c Line: 25' flowId='15']
+##teamcity[testFinished name='test.test/TestUsartModel.testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately' duration='81' flowId='15']
+##teamcity[testIgnored name='test.test/TestUsartModel.testShouldReturnWakeupMessage' flowId='15']
+##teamcity[testSuiteFinished name='TestUsartModel' flowId='15']
+```

data/plugins/report_tests_teamcity_stdout/config/defaults.yml

@@ -0,0 +1,12 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+---
+:teamcity:
+  # Override to FALSE in user, local project options to prevent CI $stdout messages
+  :build: TRUE
+...

data/plugins/report_tests_teamcity_stdout/config/report_tests_teamcity_stdout.yml

@@ -0,0 +1,11 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+---
+:plugins:
+  # tell Ceedling we got results display taken care of
+  :display_raw_test_results: FALSE

data/plugins/report_tests_teamcity_stdout/lib/report_tests_teamcity_stdout.rb

@@ -0,0 +1,163 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/plugin'
+require 'ceedling/defaults'
+
+class ReportTestsTeamcityStdout < Plugin
+
+  # `Plugin` setup()
+  def setup
+    # TEAMCITY_BUILD defaults to true but can be overridden in a user 
+    # project file to stop CI messages locally.
+    @output_enabled = TEAMCITY_BUILD
+
+    # Provide thread-safety for multi-threaded builds
+    @mutex = Mutex.new
+
+    # A counter incremented for each Ceedling test executable allowing 
+    # concurrent executables to differentiate their service messages.
+    @flowid_count = 0
+
+    # A TeamCity suite correlates to a Ceedling test executable
+    # This hash relates each test filepath to a unique Flow ID
+    # (TeamCity uses Flow IDs to differentiate messages generated in concurrent threads)
+    @suites = {}
+  end
+
+  # `Plugin` build step hook
+  def pre_test(test)
+    return if !@output_enabled
+
+    # Generate a new Flow ID and store it in test hash
+    @mutex.synchronize do
+      @flowid_count += 1
+      @suites[test] = { :flowid => @flowid_count }
+    end
+
+    # Generate first TeamCity service message
+    @mutex.synchronize do
+      teamcity_service_message(
+        "testSuiteStarted name='#{File.basename(test, '.*')}'",
+        @suites[test][:flowid]
+      )
+    end
+  end
+
+  # `Plugin` build step hook
+  def post_test_fixture_execute(arg_hash)
+    return if !@output_enabled
+
+    flowId = nil
+    test_key = arg_hash[:test_filepath]
+
+    # Get unique Flow ID
+    @mutex.synchronize do
+      flowId = @suites[test_key][:flowid]
+    end
+
+    # Gather test results for this test executable
+    results = @ceedling[:plugin_reportinator].assemble_test_results( [arg_hash[:result_file]] )
+
+    # Apportion total run time of test executable equally to each test case within it.
+    duration_ms = results[:total_time] * 1000.0
+    avg_duration = (duration_ms / [1, results[:counts][:passed] + results[:counts][:failed]].max).round
+
+    context = arg_hash[:context]
+
+    # Handle test case successes within the test executable
+    results[:successes].each do |success|
+      filepath = File.join( success[:source][:dirname], File.basename( success[:source][:basename], '.*' ) )
+
+      # puts(success)
+      success[:collection].each do |test|
+        _test = test[:test]
+
+        # Create Java-like name per TeamCity convention `package_or_namespace.ClassName.TestName`
+        #  ==> `context.TestFilepath.TestCaseName`
+        teamcity_service_message(
+          "testStarted name='#{context}.#{filepath}.#{_test}'",
+          flowId
+        )
+
+        teamcity_service_message(
+          "testFinished name='#{context}.#{filepath}.#{_test}' duration='#{avg_duration}'",
+          flowId
+        )
+      end
+    end
+
+    # Handle test case failures within the test executable
+    results[:failures].each do |failure|
+      failure[:collection].each do |test|
+        filepath = File.join( failure[:source][:dirname], File.basename( failure[:source][:basename], '.*' ) )
+
+        _test = test[:test]
+        _message = test[:message]
+
+        teamcity_service_message(
+          "testStarted name='#{context}.#{filepath}.#{_test}'",
+          flowId
+        )
+
+        # Create Java-like name per TeamCity convention `package_or_namespace.ClassName.TestName`
+        #  ==> `context.TestFilepath.TestCaseName`
+        _message = 
+          "testFailed name='#{context}.#{filepath}.#{_test}' " + 
+          "message='#{escape(_message)}' " +
+          "details='File: #{failure[:source][:file]} Line: #{test[:line]}'"
+
+        teamcity_service_message( _message, flowId )
+
+        teamcity_service_message(
+          "testFinished name='#{context}.#{filepath}.#{_test}' duration='#{avg_duration}'",
+          flowId
+        )
+      end
+    end
+
+    # Handle ignored tests
+    results[:ignores].each do |ignored|
+      ignored[:collection].each do |test|
+        filepath = File.join( ignored[:source][:dirname], File.basename( ignored[:source][:basename], '.*' ) )
+        _test = test[:test]
+
+        service_message = "testIgnored name='#{context}.#{filepath}.#{_test}'"
+
+        teamcity_service_message( service_message, flowId )
+      end
+    end
+
+  end
+
+  # `Plugin` build step hook
+  def post_test(test)
+    return if !@output_enabled
+
+    @mutex.synchronize do
+      teamcity_service_message(
+        "testSuiteFinished name='#{File.basename(test, '.*')}'",
+        @suites[test][:flowid]
+      )
+    end
+  end
+
+  ### Private
+
+  private
+
+  def escape(string)
+    # https://www.jetbrains.com/help/teamcity/service-messages.html#Escaped+Values
+    string.gsub(/['|\[\]]/, '|\0').gsub('\r', '|r').gsub('\n', '|n')
+  end
+
+  def teamcity_service_message(content, flowId=0)
+    # https://www.jetbrains.com/help/teamcity/service-messages.html
+    puts "##teamcity[#{content} flowId='#{flowId}']"
+  end
+
+end