ceedling 1.1.9 → 1.2.0

data/lib/ceedling/generators/generator_test_results_backtrace.rb

@@ -14,6 +14,16 @@ class GeneratorTestResultsBacktrace
     @RESULTS_COLLECTOR = Struct.new( :passed, :failed, :ignored, :output, keyword_init:true )
     # Alias, matching Generator's own convention for the same dependency.
     @helper = @generator_helper
+
+    # Tracks whether gdb ever explained a crash across the *whole test suite run*, not
+    # just one test executable's `do_gdb` call. `do_gdb` runs once per crashing test
+    # executable and different executables can run concurrently (see `stage_execute`'s
+    # `Batchinator#exec`), so this state is mutex-guarded. `report_gdb_attach_health`
+    # flushes it exactly once, after every test executable in the run has executed --
+    # see that method for why a per-executable summary would be the wrong shape.
+    @gdb_health_mutex = Mutex.new
+    @any_gdb_crash_in_suite = false
+    @any_useful_gdb_report_in_suite = false
   end
 
   # Re-runs each test case (or, for a parameterized test, each group of parameterized
@@ -39,6 +49,11 @@ class GeneratorTestResultsBacktrace
     # can be trusted -- see the fallback after the loop.
     any_group_crashed = false
 
+    # True once some group's crash resolves to an actual signal or assertion label --
+    # as opposed to the generic "failed to extract `gdb` report" fallback. Feeds the
+    # whole-suite accumulator below -- see `report_gdb_attach_health`.
+    any_useful_report = false
+
     # Iterate on test cases, one sub-process run per group (see `group_test_cases`)
     group_test_cases( test_cases ).each do |group|
       # Build the test fixture to run with our test case (or parameterized group) of interest
@@ -128,6 +143,8 @@ class GeneratorTestResultsBacktrace
 
           # If we found an error report line containing `test_case() at filename.c:###` in `gdb` output
           if matched
+            any_useful_report = true
+
             # Line number
             line_number = matched[1]
 
@@ -155,11 +172,19 @@ class GeneratorTestResultsBacktrace
             label = format_signal_label( crash_result[:output] )
 
             if !label.empty?
+              any_useful_report = true
+
               group_results[:output] <<
                 "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
                 " >> #{label}" \
                 "#{NEWLINE_TOKEN}(#{log_path})"
             else
+              @loginator.log(
+                "`gdb` produced no usable diagnostics for crashed test case `#{test_case[:test]}` " \
+                "-- see #{log_path}",
+                Verbosity::COMPLAIN
+              )
+
               group_results[:output] <<
                 "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
                 "Test case crashed (failed to extract `gdb` report)" \
@@ -208,6 +233,14 @@ class GeneratorTestResultsBacktrace
       return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
     end
 
+    # Record this executable's contribution to the whole suite's gdb health --
+    # `report_gdb_attach_health` flushes the actual summary once, after every test
+    # executable in the run has executed.
+    @gdb_health_mutex.synchronize do
+      @any_gdb_crash_in_suite = true
+      @any_useful_gdb_report_in_suite ||= any_useful_report
+    end
+
     # Reset shell result exit code and output
     shell_result[:exit_code] = test_case_results[:failed]
     shell_result[:output] =
@@ -221,142 +254,198 @@ class GeneratorTestResultsBacktrace
     return shell_result
   end
 
+  # Flushes the whole-suite gdb health check accumulated across every `do_gdb` call in
+  # this run. Call once, after every test executable in the suite has finished
+  # executing and before final results reporting.
+  #
+  # A crash that gdb never explained is worth a summary at the single-test-case level
+  # (see the `Verbosity::COMPLAIN` log inside `do_gdb`), but a crash that gdb never
+  # explained *for any test executable in the whole run* is a materially different,
+  # stronger signal: gdb itself is the more likely problem, not an unusually
+  # hard-to-diagnose crash (e.g. it lost the ability to attach to a process). That
+  # deserves its own summary, once per suite, not once per crashing test executable.
+  def report_gdb_attach_health()
+    @gdb_health_mutex.synchronize do
+      return unless @any_gdb_crash_in_suite
+      return if @any_useful_gdb_report_in_suite
+
+      @loginator.log(
+        "`gdb` produced no usable diagnostics for any crash in this test suite run " \
+        "-- validate that `gdb` has permissions to attach to processes and check `gdb`'s own results directly.",
+        Verbosity::ERRORS, LogLabels::WARNING
+      )
+
+      @any_gdb_crash_in_suite = false
+      @any_useful_gdb_report_in_suite = false
+    end
+  end
+
   # Re-runs each test case (or, for a parameterized test, each group of parameterized
   # cases -- see `group_test_cases`) individually to determine which one(s) crashed.
   # For crash cases, captures any extra output from the test binary (e.g.
   # assertion messages on stderr) and includes it in the failure report.
   # Returns a modified shell_result with regenerated output.
-  def do_simple(filename, executable, shell_result, test_cases, context:)
-    # Clean stats tracker
-    test_case_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
-
-    # Reset time
-    shell_result[:time] = 0
-
-    # True once some retry group has actually shown crash evidence of its own -- a
-    # member with no result line, or (below) a group whose real status contradicts a
-    # fully clean set of matches. If this stays false across every group, the whole
-    # diagnostic never reproduced or attributed the crash the main run already
-    # detected, and none of it can be trusted -- see the fallback after the loop.
-    any_group_crashed = false
+  #
+  # `any_crashed` turns true once some retry group shows crash evidence of its own. If it
+  # stays false across every group, the diagnostic never reproduced or attributed the crash
+  # the main run already detected, and none of it can be trusted.
+  #
+  # `tool` is the fixture tool the main run used. A swapped tool marked `:backtrace_rerun`
+  # (e.g. one setting a sanitizer's runtime options) runs the reruns too. Otherwise reruns
+  # go through the dedicated backtrace tool, so a wrapper that writes one report per test,
+  # such as Valgrind's, keeps the main run's report.
+  #
+  # `explain` names the cause of a rerun's crash, if a plugin recognizes it.
+  def do_simple(filename, executable, shell_result, test_cases, context:, tool: nil, explain: nil)
+    results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
+    shell_result[:time] = 0 # Summed over the per-group sub-process runs
+    any_crashed = false
 
-    # Iterate on test cases, one sub-process run per group (see `group_test_cases`)
     group_test_cases( test_cases ).each do |group|
-      # Build the test fixture to run with our test case (or parameterized group) of interest
-      command = @tool_executor.build_command_line(
-        @configurator.tools_test_fixture_simple_backtrace, [],
-        executable,
-        unity_filter_arg( group )
-      )
-      # Things are gonna go boom, so ignore booms to get output
-      command[:options][:boom] = false
+      group_results, crashed = simple_group_results( filename, executable, group, shell_result, { tool: tool, explain: explain } )
+      any_crashed ||= crashed
+      merge_results( results, group_results )
+    end
 
-      crash_result = @tool_executor.exec( command )
+    return simple_crash_not_reproduced( filename, executable, shell_result, test_cases ) unless any_crashed
 
-      # Sum execution time for each sub-process run
-      # Note: Running tests separately increases total execution time
-      shell_result[:time] += crash_result[:time].to_f()
+    return regenerate_simple_results( shell_result, test_cases, results )
+  end
 
-      # Buffered separately from test_case_results and only merged in afterward, since
-      # the status check below can still discard every match here in favor of a crash
-      # attribution, once the whole group has been seen.
-      group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
-      crashed = false # Has the actual crash in this group already been attributed?
+  ### Private ###
+  private
 
-      # Attribute each group member its own real result line, if Unity printed one
-      group.each do |test_case|
-        case crash_result[:output]
-        # Success test case -- tolerates a trailing duration suffix (e.g. `PASS (12 ms)`,
-        # UNITY_INCLUDE_EXEC_TIME), matching the real format generator_test_results.rb's
-        # own parser already recognizes for it.
-        when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:PASS(\s\(.*ms\))?\s*$)/
-          group_results[:passed]  += 1
-          group_results[:output] << $1
+  # One sub-process run for a group. Its results are buffered apart from the file's, since
+  # the status check can still discard every match in favor of a crash attribution once the
+  # whole group has been seen.
+  # `rerun` holds the fixture tool and crash explainer do_simple was given
+  def simple_group_results(filename, executable, group, shell_result, rerun)
+    crash_result = run_simple_group( executable, group, rerun[:tool] )
+    # Running tests separately increases total execution time
+    shell_result[:time] += crash_result[:time].to_f()
+
+    explanation = rerun[:explain]&.call( crash_result )
+    group_results, crashed = attribute_group_results( filename, group, crash_result[:output], explanation )
+    return [group_results, crashed] if crashed
+
+    # Every member matched a result line, but a retry through its own separately configured
+    # tool can complete cleanly and print a legitimate-looking PASS even when it is no
+    # trustworthy stand-in for what happened. The real-status check the main run relies on
+    # applies here too.
+    return [group_results, false] unless @helper.test_crash?( filename, executable, crash_result )
+
+    return [contradicted_group_results( filename, group ), true]
+  end
 
-        # Ignored test case -- tolerates a trailing message (TEST_IGNORE_MESSAGE), same as
-        # FAIL below. Before this fix, a message-carrying ignore fell through to "unresolved"
-        # and was misreported as crash evidence -- IGNORE was only recognized bare.
-        when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:IGNORE(:.+)?\s*$)/
-          group_results[:ignored] += 1
-          group_results[:output] << $1
+  def run_simple_group(executable, group, tool)
+    command = simple_rerun_command( executable, unity_filter_arg( group ), tool )
+    # Things are gonna go boom, so ignore booms to get output
+    command[:options][:boom] = false
 
-        when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:FAIL(:.+)?\s*$)/
-          group_results[:failed]  += 1
-          group_results[:output] << $1
+    return @tool_executor.exec( command )
+  end
 
-        # No result line for this member -- either it crashed, or it never got to run
-        # because an earlier member in this same group crashed.
-        else
-          group_results[:failed] += 1
+  # The dedicated backtrace tool places the filter as its own argument. A fixture tool runs
+  # the executable itself, so the filter follows its arguments.
+  def simple_rerun_command(executable, filter, tool)
+    unless tool&.dig( :backtrace_rerun )
+      return @tool_executor.build_command_line( @configurator.tools_test_fixture_simple_backtrace, [], executable, filter )
+    end
 
-          if crashed
-            group_results[:output] <<
-              "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
-              "Test case not run -- an earlier case in this parameterized test group crashed"
-          else
-            crashed = true
-            # Collect any non-result, non-blank lines (e.g. assertion messages on stderr)
-            extra = extract_simple_crash_output( crash_result[:output], filename )
-            test_output = "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed"
-            test_output += " >> #{extra.join(NEWLINE_TOKEN)}" unless extra.empty?
-            group_results[:output] << test_output
-          end
-        end
-      end
+    return @tool_executor.build_command_line( tool, [], executable, trailing: [filter] )
+  end
 
-      # Every member in this group resolved via a matched result line -- but a diagnostic
-      # retry running through its own, separately-configured tool (e.g. without the main
-      # :test_fixture's sanitizer options) can complete cleanly and print a legitimate-
-      # looking PASS even when it isn't a trustworthy stand-in for what actually happened.
-      # The same real-status check the main run itself relies on applies here too.
-      if !crashed && @helper.test_crash?( filename, executable, crash_result )
-        crashed = true
-        group_results = @RESULTS_COLLECTOR.new(
-          passed: 0, ignored: 0, failed: group.size,
-          output: group.map { |test_case|
-            "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
-            " >> diagnostic retry's own exit status contradicts its reported clean result"
-          }
-        )
-      end
+  # Gives each group member its own real result line, if Unity printed one. The first member
+  # with none is the crash; any after it never ran because that earlier member crashed.
+  def attribute_group_results(filename, group, output, explanation)
+    group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
+    crashed = false
 
-      any_group_crashed ||= crashed
+    group.each do |test_case|
+      outcome, line = simple_result_line( filename, test_case, output )
+      outcome, line = unresolved_case( filename, test_case, output, crashed, explanation ) if outcome.nil?
+      crashed = true if outcome == :crashed
 
-      test_case_results[:passed]  += group_results[:passed]
-      test_case_results[:ignored] += group_results[:ignored]
-      test_case_results[:failed]  += group_results[:failed]
-      test_case_results[:output].concat( group_results[:output] )
+      group_results[outcome == :crashed ? :failed : outcome] += 1
+      group_results[:output] << line
     end
 
-    # No retry group, across the entire file, ever showed real crash evidence, yet this
-    # method only runs because the main run was already established as a crash. An
-    # all-clean diagnostic must never be allowed to overrule that -- fall back to the
-    # same whole-file crash-as-failure construction the :none backtrace setting uses.
-    unless any_group_crashed
-      @loginator.log(
-        "Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
-        "already detected on the main run -- reporting the original crash rather than trusting " \
-        "an all-clear diagnostic that contradicts it.",
-        Verbosity::ERRORS, LogLabels::CRASH
-      )
-      return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
+    return [group_results, crashed]
+  end
+
+  # A PASS tolerates a trailing duration suffix (e.g. `PASS (12 ms)`, UNITY_INCLUDE_EXEC_TIME),
+  # matching what generator_test_results.rb's parser recognizes. IGNORE and FAIL tolerate a
+  # trailing message (TEST_IGNORE_MESSAGE).
+  def simple_result_line(filename, test_case, output)
+    prefix = "^#{Regexp.escape(filename)}:\\d+:#{Regexp.escape(test_case[:test])}"
+
+    case output
+    when /(#{prefix}:PASS(\s\(.*ms\))?\s*$)/ then return [:passed, $1]
+    when /(#{prefix}:IGNORE(:.+)?\s*$)/      then return [:ignored, $1]
+    when /(#{prefix}:FAIL(:.+)?\s*$)/        then return [:failed, $1]
     end
 
-    # Reset shell result exit code and output
-    shell_result[:exit_code] = test_case_results[:failed]
+    return [nil, nil]
+  end
+
+  def unresolved_case(filename, test_case, output, crashed, explanation)
+    location = "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: "
+    return [:failed, location + "Test case not run -- an earlier case in this parameterized test group crashed"] if crashed
+
+    # A named cause is where the crash's full report lives
+    return [:crashed, location + explanation] if explanation
+
+    # Any non-result, non-blank lines (e.g. assertion messages on stderr)
+    extra = extract_simple_crash_output( output, filename )
+    line  = location + "Test case crashed"
+    line += " >> #{extra.join(NEWLINE_TOKEN)}" unless extra.empty?
+
+    return [:crashed, line]
+  end
+
+  def contradicted_group_results(filename, group)
+    return @RESULTS_COLLECTOR.new(
+      passed: 0, ignored: 0, failed: group.size,
+      output: group.map { |test_case|
+        "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
+        " >> diagnostic retry's own exit status contradicts its reported clean result"
+      }
+    )
+  end
+
+  def merge_results(results, group_results)
+    results[:passed]  += group_results[:passed]
+    results[:ignored] += group_results[:ignored]
+    results[:failed]  += group_results[:failed]
+    results[:output].concat( group_results[:output] )
+  end
+
+  # This method only runs because the main run was established as a crash, so an all-clean
+  # diagnostic must never overrule it. The fallback is the same whole-file crash-as-failure
+  # construction the :none backtrace setting uses.
+  def simple_crash_not_reproduced(filename, executable, shell_result, test_cases)
+    @loginator.log(
+      "Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
+      "already detected on the main run -- reporting the original crash rather than trusting " \
+      "an all-clear diagnostic that contradicts it.",
+      Verbosity::ERRORS, LogLabels::CRASH
+    )
+    return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
+  end
+
+  def regenerate_simple_results(shell_result, test_cases, results)
+    shell_result[:exit_code] = results[:failed]
     shell_result[:output] =
       @generator_test_results.regenerate_test_executable_stdout(
         total:   test_cases.size(),
-        ignored: test_case_results[:ignored],
-        failed:  test_case_results[:failed],
-        output:  test_case_results[:output]
+        ignored: results[:ignored],
+        failed:  results[:failed],
+        output:  results[:output]
       )
 
     return shell_result
   end
 
-  ### Private ###
-  private
 
   # Groups test cases so each parameterized test's cases are isolated together as one
   # sub-process run instead of one run per case. Unity's `-n`/`-f` command-line filter
@@ -457,10 +546,13 @@ class GeneratorTestResultsBacktrace
   # lines (PASS/FAIL/IGNORE) and not blank. These are typically stderr output
   # from the crashed test binary — assertion messages, abort text, etc.
   def extract_simple_crash_output(output, filename)
+    # Compile once rather than once per line of output
+    result_line = /^#{Regexp.escape(filename)}.+:(PASS|FAIL|IGNORE)/
+
     output.lines.filter_map do |line|
       line = line.strip
       next if line.empty?
-      next if line =~ /^#{Regexp.escape(filename)}.+:(PASS|FAIL|IGNORE)/
+      next if line.match?( result_line )
       line
     end
   end

data/lib/ceedling/generators/generator_test_results_sanity_checker.rb

@@ -56,7 +56,7 @@ class GeneratorTestResultsSanityChecker
   
   def sanity_check_warning(file, message)
     unless defined?(CEEDLING_IGNORE_SANITY_CHECK)
-      notice = "Internal sanity check for test fixture '#{file.ext(@configurator.extension_executable)}' finds that #{message}\n" +
+      notice = "Internal sanity check for test fixture '#{file.ext(@configurator.extension_executable.primary)}' finds that #{message}\n" +
                "  Possible causes:\n" +
                "    1. Your test + source dereferenced a null pointer.\n" +
                "    2. Your test + source indexed past the end of a buffer.\n" +

data/lib/ceedling/includes/include_factory.rb

@@ -18,13 +18,9 @@ class IncludeFactory
     return nil
   end
 
-  def user_include_from_filepath(filepath)
+  def user_include_from_filepath(filepath, test: nil)
     if File.basename(filepath).start_with?( @configurator.cmock_mock_prefix )
-      # Remove any build directory path that snuck into mock handling.
-      # This can happen from discovering an empty mock stand-in or previously generated mock files.
-      # This regex matches the base build mocks directory and any test name subdirectory beneath it.
-      _filepath = filepath.sub( /^#{Regexp.escape( @configurator.cmock_mock_path )}\/[^\/]+\//, '' )
-      return MockInclude.new(_filepath)
+      return MockInclude.new( strip_mock_test_subdir( filepath, test ) )
     end
     return UserInclude.new(filepath)
   end
@@ -40,4 +36,23 @@ class IncludeFactory
     return SystemInclude.new(filepath)
   end
 
+  private
+
+  # A resolved mock include's filepath may carry a leading build directory path that
+  # snuck in from discovering an empty mock stand-in or an already-generated mock file.
+  # That leading path is the current test's own mock subdirectory -- which mirrors the
+  # test's path below its configured :paths -> :test root, so it can be more than one
+  # segment deep (e.g. `adc/TestFoo`) -- followed by however many further segments mirror
+  # the mocked header's own relative directory (e.g. `calculators/`). Knowing the current
+  # test's identity lets exactly the first part be removed, leaving any header-mirroring
+  # segments after it untouched. Without that identity, fall back to stripping everything
+  # up through the last segment, the best a caller lacking test context can do.
+  def strip_mock_test_subdir(filepath, test)
+    if test
+      prefix = File.join( @configurator.cmock_mock_path, test.to_s ) + '/'
+      return filepath.start_with?( prefix ) ? filepath[prefix.length..] : filepath
+    end
+    return filepath.sub( /^#{Regexp.escape( @configurator.cmock_mock_path )}\/.+\//, '' )
+  end
+
 end

data/lib/ceedling/includes/include_pathinator.rb

@@ -7,6 +7,7 @@
 
 require 'pathname'
 require 'ceedling/exceptions'
+require 'ceedling/file_path_utils'
 
 class IncludePathinator
 
@@ -43,8 +44,10 @@ class IncludePathinator
     # Get all paths specified by TEST_INCLUDE_PATH() directive in test files
     directive_paths = @extractor.lookup_all_include_paths
 
-    # Add to collection of headers (Rake FileList) with directive paths and shallow wildcard matching on header file extension
-    headers += @file_wrapper.instantiate_file_list( directive_paths.map { |path| File.join(path, '*' + EXTENSION_HEADER) } )
+    # Add to collection of headers (Rake FileList) with directive paths and shallow wildcard
+    # matching on header file extension -- a project may configure more than one header
+    # extension, so each directive path contributes one glob pattern per configured extension
+    headers += @file_wrapper.instantiate_file_list( directive_paths.flat_map { |path| EXTENSION_HEADER.glob_patterns(path) } )
     headers.resolve()
 
     headers.uniq!
@@ -59,6 +62,39 @@ class IncludePathinator
     return headers
   end
 
+  # Header files reachable from an already-ordered directory priority list (e.g. a
+  # test's own search_paths), in that same order and deduplicated by first occurrence.
+  # This is the candidate list a compiler's own -I search would consult for a single
+  # test's #include resolution -- distinct from collection_all_headers, which is one
+  # global, project-wide list blind to any one test's own TEST_INCLUDE_PATH() priority.
+  def ordered_header_files(search_paths)
+    headers = []
+    search_paths.each do |path|
+      headers += @file_wrapper.directory_listing( @configurator.extension_header.glob_patterns(path) )
+    end
+    return headers.uniq
+  end
+
+  # The project's whole header collection, reordered so headers beneath this test's
+  # own earlier search paths come first.
+  #
+  # Distinct from ordered_header_files above, which lists each search path's immediate
+  # contents the way a compiler's own non-recursive -I search does. A reference naming
+  # a header by path needs the whole collection instead, since the file it names can
+  # sit at any depth below a search path, and it needs this test's own priority so a
+  # TEST_INCLUDE_PATH() decides among same-named candidates.
+  #
+  # Ordering within one priority rank is the collection's own, which is what keeps a
+  # reference that distinguishes nothing resolving exactly as it did before.
+  def prioritized_header_files(search_paths)
+    roots = search_paths.map { |path| FilePathUtils.no_decorators( path ) }.reject(&:empty?)
+
+    @configurator.collection_all_headers.to_a.each_with_index.sort_by do |header, index|
+      rank = roots.index { |root| header == root || header.start_with?( root + '/' ) } || roots.length
+      [rank, index]
+    end.map(&:first)
+  end
+
   def augment_environment_header_files(headers)
     @configurator.redefine_element(:collection_all_headers, headers)
   end
@@ -69,11 +105,11 @@ class IncludePathinator
     return @extractor.lookup_include_paths_list(filepath)
   end
 
-  # Gather together [:paths][:test] that actually contain .h files
+  # Gather together [:paths][:test] that actually contain header files
   def collect_test_include_paths
     paths = []
     @configurator.collection_paths_test.each do |path|
-      headers = @file_wrapper.directory_listing( File.join( path, '*' + @configurator.extension_header ) )
+      headers = @file_wrapper.directory_listing( @configurator.extension_header.glob_patterns(path) )
       paths << path if headers.length > 0
     end