ceedling 1.1.10 → 1.2.1

data/plugins/valgrind/lib/valgrind.rb

@@ -6,26 +6,23 @@
 # =========================================================================
 
 require 'ceedling/plugins/plugin'
-require 'fileutils'
 require 'ceedling/constants'
+require 'ceedling/exceptions'
 require 'valgrind_constants'
+require 'valgrind_report'
 
 class Valgrind < Plugin
 
   def setup
     @memory_errors   = 0
     @tests_processed = 0
+    @reports_read    = 0
+    @explained       = []
     @mutex = Mutex.new()
 
     @result_list = []
 
-    # Validate the Valgrind tool upfront
-    @ceedling[:tool_validator].validate(
-      tool: TOOLS_VALGRIND,
-      boom: true
-    )
-
-    FileUtils.mkdir_p(VALGRIND_ARTIFACTS_PATH)
+    @environment_validated = false
 
     # Aliases
     @configurator        = @ceedling[:configurator]
@@ -34,9 +31,71 @@ class Valgrind < Plugin
     @rake_invocation_tracker   = @ceedling[:rake_invocation_tracker]
     @plugin_manager      = @ceedling[:plugin_manager]
     @plugin_reportinator = @ceedling[:plugin_reportinator]
+    @dependinator        = @ceedling[:dependinator]
+    @file_wrapper        = @ceedling[:file_wrapper]
+    @system_wrapper      = @ceedling[:system_wrapper]
+
+    # Cloned from the original, unflattened project config (project_config_hash
+    # has none of this -- flattening leaves no nested :valgrind key) so the whole
+    # section can ride along as dependency-tracker meta alongside the specific
+    # :arguments/:tools tracking already baked into the tool hash below -- the
+    # same belt-and-suspenders reasoning :cmock's own whole-section meta capture
+    # already uses.
+    @valgrind_config = @ceedling[:setupinator].config_hash[:valgrind].clone
   end
 
-  def pre_test_fixture_execute(arg_hash)
+  # Confirms Valgrind can actually run here, then creates the artifacts directory.
+  # Called externally by the plugin Rakefile's :valgrind_deps task, which every
+  # `valgrind:` task depends on.
+  #
+  # Validation is deliberately not done in setup(). setup() runs for every Ceedling
+  # command this plugin is merely enabled for, so validating there makes Valgrind a
+  # hard dependency of plain `test:all`. Valgrind does not exist on Windows at all,
+  # which would leave such a project unable to run any Ceedling command.
+  #
+  # Memoized. Every `valgrind:` task funnels through the same Rake dependency.
+  def validate_environment!
+    return if @environment_validated
+
+    if @system_wrapper.windows?
+      raise CeedlingException.new(
+        "Valgrind does not run on Windows.\n" \
+        "Remove `valgrind` from :plugins ↳ :enabled, or build in a Linux environment."
+      )
+    end
+
+    begin
+      @ceedling[:tool_validator].validate( tool: TOOLS_VALGRIND, boom: true )
+    rescue CeedlingException => ex
+      # The bare validator message only reports a missing executable. Name the
+      # platform constraint too, since that is the usual reason it is missing.
+      raise CeedlingException.new(
+        "#{ex.message}\n" \
+        "NOTE: Valgrind runs on Linux and other Unix-like systems. macOS support is limited."
+      )
+    end
+
+    @file_wrapper.mkdir( VALGRIND_ARTIFACTS_PATH )
+
+    @environment_validated = true
+  end
+
+  # Swaps in the per-test Valgrind-wrapped fixture tool ahead of TestBuildExecutor's
+  # own dependency-tracker meta capture, so a change to :valgrind ↳ :arguments (or
+  # :tools ↳ :valgrind itself) is what actually forces a rerun -- not the plain test
+  # fixture tool this replaces.
+  # A sanitizer-instrumented executable can't run under Valgrind, so a valgrind: build whose
+  # flags carry -fsanitize stops. The flags often come from :flags ↳ :test, which a valgrind
+  # build uses for any operation it gives no flags of its own.
+  def pre_test_compile_register(arg_hash)
+    refuse_sanitizer_flags( arg_hash, 'compile' ) if arg_hash[:context] == VALGRIND_SYM
+  end
+
+  def pre_test_link_register(arg_hash)
+    refuse_sanitizer_flags( arg_hash, 'link' ) if arg_hash[:context] == VALGRIND_SYM
+  end
+
+  def pre_test_fixture_register(arg_hash)
     return unless arg_hash[:context] == VALGRIND_SYM
 
     @mutex.synchronize do
@@ -44,19 +103,21 @@ class Valgrind < Plugin
     end
 
     test_name = arg_hash[:test_name]
-    log_path  = File.join( VALGRIND_ARTIFACTS_PATH, "#{test_name}.log" )
-
-    # Replace the test fixture tool with a per-test valgrind wrapper.
-    # Arguments come from :valgrind ↳ :arguments project config.
-    # ${1} (the test executable substitution token) is always appended by the plugin,
-    # not user-visible in config.
-    valgrind_args = Array( @configurator.project_config_hash[:valgrind_arguments] )
-    arg_hash[:tool] = {
-      :executable => TOOLS_VALGRIND[:executable],
-      :name       => TOOLS_VALGRIND[:name],
-      :optional   => TOOLS_VALGRIND[:optional],
-      :arguments  => ["--log-file=\"#{log_path}\""] + valgrind_args + ["${1}"],
-    }
+
+    # Merged onto the configured tool rather than assembled from scratch. Any other
+    # key a user set under :tools ↳ :valgrind, such as :stderr_redirect, survives.
+    arg_hash[:tool] = TOOLS_VALGRIND.merge( :arguments => build_arguments( test_name ) )
+
+    # The whole :valgrind config as meta, redundant alongside :arguments already
+    # being part of the tool hash above -- belt-and-suspenders against a future
+    # :valgrind key affecting a run this file doesn't yet know to thread through by
+    # hand. Additive: merges with TestBuildExecutor's own register call for this
+    # same target moments later.
+    @dependinator.register( arg_hash[:target], meta: { valgrind: @valgrind_config } )
+  end
+
+  def pre_test_fixture_execute(arg_hash)
+    return unless arg_hash[:context] == VALGRIND_SYM
 
     msg = "Running #{File.basename(arg_hash[:executable])} under Valgrind"
     arg_hash[:msg] = @reportinator.generate_progress( msg )
@@ -65,27 +126,39 @@ class Valgrind < Plugin
   def post_test_fixture_execute(arg_hash)
     return unless arg_hash[:context] == VALGRIND_SYM
 
-    result_file = arg_hash[:result_file]
+    collect_result( arg_hash[:result_file] )
+
+    test_name = arg_hash[:test_name]
+    errors    = count_errors( test_name )
+
+    # nil means no report could be read. count_errors already warned.
+    return if errors.nil?
 
     @mutex.synchronize do
-      if (result_file =~ /#{PROJECT_TEST_RESULTS_PATH}/) && !@result_list.include?(result_file)
-        @result_list << arg_hash[:result_file]
-      end
+      @reports_read += 1
+      @memory_errors += errors
     end
 
+    # A crash Valgrind explained has its errors named in the crash notice
+    return if errors == 0 || @mutex.synchronize { @explained.include?( test_name ) }
+
+    msg = "Valgrind detected #{pluralize(errors, 'memory error')} in #{test_name} ➡️ see #{report_filepath(test_name)}"
+    @loginator.log( msg, Verbosity::ERRORS, LogLabels::ERROR )
+  end
+
+  # Only the main run ran under Valgrind. A backtrace rerun runs the bare executable.
+  def explain_test_crash(arg_hash)
+    return unless arg_hash[:context] == VALGRIND_SYM && arg_hash[:crash_notice]
+
     test_name = arg_hash[:test_name]
-    log_path  = File.join( VALGRIND_ARTIFACTS_PATH, "#{test_name}.log" )
-    return unless File.exist?( log_path )
+    path      = report_filepath( test_name )
+    return unless @file_wrapper.exist?( path )
 
-    log_content = File.read( log_path )
-    if (m = log_content.match( /ERROR SUMMARY:\s+(\d+) errors?/ )) && m[1].to_i > 0
-      @mutex.synchronize do
-        @memory_errors += m[1].to_i
-      end
-      msg = "Valgrind detected #{pluralize(m[1].to_i, 'memory error')} in #{test_name} ➡️ see #{log_path}"
-      @loginator.log( msg, Verbosity::ERRORS, LogLabels::ERROR )
-      log_results()
-    end
+    cause = ValgrindReport.first_error( @file_wrapper.read( path ), xml: xml_report_enabled? )
+    return if cause.nil?
+
+    @mutex.synchronize { @explained << test_name }
+    arg_hash.merge!( explanation_heading: 'terminated with Valgrind memory errors', explanation: cause, explanation_details: ["Boom log: #{path}"] )
   end
 
   def post_build(_timestamp_s)
@@ -107,7 +180,10 @@ class Valgrind < Plugin
       @plugin_reportinator.run_test_results_report( hash, verbosity )
     end
 
+    # Reported once, here. Emitting this per erroring test repeated the same summary
+    # several times with a different partial count each time.
     log_results()
+
     if @configurator.valgrind_fail_build && @memory_errors > 0
       msg = "Valgrind detected #{pluralize(@memory_errors, 'memory error')} across #{pluralize(@tests_processed, 'test')}"
       @plugin_manager.register_build_failure( VALGRIND_SYM, msg )
@@ -116,10 +192,97 @@ class Valgrind < Plugin
 
   private
 
+  # `[`/`]` are legal filename characters but regex character-class syntax, so the path is
+  # escaped. A literal bracket in it (e.g. in :build_root:) would otherwise break the match,
+  # and the test would go uncounted.
+  def collect_result(result_file)
+    @mutex.synchronize do
+      @result_list << result_file if result_file =~ /#{Regexp.escape(VALGRIND_RESULTS_PATH)}/ && !@result_list.include?( result_file )
+    end
+  end
+
+  def refuse_sanitizer_flags(arg_hash, operation)
+    sanitizing = Array( arg_hash[:flags] ).grep( /\A-fsanitize/ )
+    return if sanitizing.empty?
+
+    raise CeedlingException.new(
+      "Valgrind cannot run a test executable built with sanitizers, but this valgrind: build's " \
+      "#{operation} flags include #{sanitizing.map { |flag| "`#{flag}`" }.join( ', ' )}. " \
+      "Give the valgrind build #{operation} flags of its own under :flags ↳ :valgrind, or move the " \
+      "sanitizer flags out of :flags ↳ :test."
+    )
+  end
+
+  def xml_report_enabled?
+    return @configurator.project_config_hash[:valgrind_xml_report] == true
+  end
+
+  def log_filepath(test_name)
+    return File.join( VALGRIND_ARTIFACTS_PATH, "#{test_name}.log" )
+  end
+
+  def xml_filepath(test_name)
+    return File.join( VALGRIND_ARTIFACTS_PATH, "#{test_name}.xml" )
+  end
+
+  # The report a user should consult, and the one errors are counted from.
+  def report_filepath(test_name)
+    return xml_report_enabled? ? xml_filepath( test_name ) : log_filepath( test_name )
+  end
+
+  # Assembles the full Valgrind argument list for one test.
+  #
+  # Suppression files lead so a user's own :arguments entries can still override
+  # anything they set. `${1}` is the test executable substitution token and is always
+  # last. Neither is user-visible in :valgrind configuration.
+  def build_arguments(test_name)
+    config = @configurator.project_config_hash
+
+    arguments = Array( config[:valgrind_suppressions] ).map { |file| "--suppressions=\"#{file}\"" }
+
+    arguments << "--log-file=\"#{log_filepath( test_name )}\""
+
+    if xml_report_enabled?
+      arguments << '--xml=yes'
+      arguments << "--xml-file=\"#{xml_filepath( test_name )}\""
+    end
+
+    arguments += Array( config[:valgrind_arguments] )
+    arguments << '${1}'
+
+    return arguments
+  end
+
+  # Returns the memory error count for one test, or nil when no report could be read.
+  #
+  # Which report is authoritative depends on :xml_report. `--xml=yes` leaves the text
+  # log empty, so XML is the only place errors appear once it is enabled.
+  def count_errors(test_name)
+    path = report_filepath( test_name )
+
+    unless @file_wrapper.exist?( path )
+      # A silent skip here reads exactly like a clean result. It is not one. The
+      # memory error status of this test is unknown.
+      msg = "Valgrind wrote no report for #{test_name} at #{path}. Memory error status is unknown."
+      @loginator.log( msg, Verbosity::COMPLAIN, LogLabels::WARNING )
+      return nil
+    end
+
+    contents = @file_wrapper.read( path )
+
+    # One <error> element per finding. This total matches the text report's own
+    # ERROR SUMMARY count for the same run.
+    return contents.scan( /<error\b/ ).length if xml_report_enabled?
+
+    # Valgrind writes one ERROR SUMMARY per traced process. A test that forks
+    # produces several, so every one is summed. Reading only the first undercounts.
+    return contents.scan( /ERROR SUMMARY:\s+(\d+) errors?/ ).flatten.map( &:to_i ).sum
+  end
+
   def log_results
     @mutex.synchronize do
-      if @tests_processed > 0
-        msg = "\nWrote #{pluralize(@tests_processed, 'Valgrind report')} to #{VALGRIND_ARTIFACTS_PATH}/"
+      if @reports_read > 0
+        msg = "\nWrote #{pluralize(@reports_read, 'Valgrind report')} to #{VALGRIND_ARTIFACTS_PATH}/"
         @loginator.log( msg, Verbosity::NORMAL, LogLabels::NOTICE )
       end
     end

data/plugins/valgrind/lib/valgrind_constants.rb

@@ -10,4 +10,8 @@ VALGRIND_ROOT_NAME              = 'valgrind'.freeze
 VALGRIND_TASK_ROOT              = VALGRIND_ROOT_NAME + ':'
 VALGRIND_SYM                    = VALGRIND_ROOT_NAME.to_sym unless defined?(VALGRIND_SYM)
 
+VALGRIND_BUILD_PATH             = File.join(PROJECT_BUILD_ROOT, VALGRIND_ROOT_NAME)
 VALGRIND_ARTIFACTS_PATH         = File.join(PROJECT_BUILD_ARTIFACTS_ROOT, VALGRIND_ROOT_NAME)
+
+# A valgrind: build writes its test results apart from an ordinary test build's
+VALGRIND_RESULTS_PATH           = File.join(VALGRIND_BUILD_PATH, BUILD_RESULTS_DIR)

data/plugins/valgrind/lib/valgrind_report.rb

@@ -0,0 +1,52 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'cgi'
+
+# The first memory error in a Valgrind report, named by its description and the first
+# stack frame that has a source location
+module ValgrindReport
+
+  # In a text log, an error's description line is followed at once by its first frame.
+  # Valgrind's header and termination notices are not.
+  TEXT_ERROR = /^==\d+== (\S[^\n]*)\n((?:==\d+==\s+(?:at|by) 0x\h+:[^\n]*\n)+)/
+  TEXT_FRAME = /\(([^()\s]+:\d+)\)$/
+
+  def self.first_error(contents, xml:)
+    return xml ? from_xml( contents ) : from_text( contents )
+  end
+
+  def self.from_text(contents)
+    match = contents.match( TEXT_ERROR )
+    return nil if match.nil?
+
+    frame = match[2].lines.map { |line| line.chomp[TEXT_FRAME, 1] }.compact.first
+    return describe( match[1], frame )
+  end
+
+  # A leak's description sits in <xwhat><text> where other errors use <what>
+  def self.from_xml(contents)
+    error = contents[%r{<error>.*?</error>}m]
+    return nil if error.nil?
+
+    what  = error[%r{<what>(.*?)</what>}m, 1] || error[%r{<xwhat>\s*<text>(.*?)</text>}m, 1]
+    frame = error.scan( %r{<frame>(.*?)</frame>}m ).flatten.map { |body| xml_location( body ) }.compact.first
+    return what && describe( CGI.unescapeHTML( what ), frame )
+  end
+
+  def self.xml_location(frame)
+    file = frame[%r{<file>(.*?)</file>}m, 1]
+    line = frame[%r{<line>(\d+)</line>}m, 1]
+    return file && line && "#{file}:#{line}"
+  end
+
+  def self.describe(what, location)
+    return location ? "#{what} at #{location}" : what
+  end
+
+  private_class_method :from_text, :from_xml, :xml_location, :describe
+end

data/plugins/valgrind/valgrind.rake

@@ -11,42 +11,63 @@
 ## to avoid marking test: tasks as invoked.
 ##
 
-CLEAN.include(File.join(VALGRIND_ARTIFACTS_PATH, '*'))
+# The plugin's own build context, kept apart from test builds
+CLEAN.include(File.join(VALGRIND_BUILD_PATH, BUILD_OUT_DIR, '*'))
+CLEAN.include(File.join(VALGRIND_BUILD_PATH, BUILD_RESULTS_DIR, '*'))
+CLEAN.include(File.join(VALGRIND_BUILD_PATH, BUILD_DEPENDENCIES_DIR, '*'))
+CLEAN.include(File.join(VALGRIND_ARTIFACTS_PATH, '**/*'))
+
+CLOBBER.include(File.join(VALGRIND_BUILD_PATH, '**/*'))
+
+# Declares the artifacts path as a Rake directory task so the `directories`
+# prerequisite below can actually resolve it. Without this the prerequisite only
+# resolved because the directory already happened to exist on disk.
+directory(VALGRIND_ARTIFACTS_PATH)
 
 task directories: [VALGRIND_ARTIFACTS_PATH]
 
-task :valgrind => [:prepare] do
+# Every valgrind: task depends on this. It is the one place the Valgrind executable
+# and the host platform are checked, so a build that never runs a valgrind: task
+# never requires Valgrind to be installed. See Valgrind#validate_environment!.
+task :valgrind_deps do
+  @ceedling[VALGRIND_SYM].validate_environment!
+end
+
+task :valgrind => [:prepare, :valgrind_deps] do
   Rake.application['valgrind:all'].invoke
 end
 
 namespace VALGRIND_SYM do
 
   desc "Run all unit tests under Valgrind (also just 'valgrind' works)."
-  task :all => [:prepare] do
+  task :all => [:prepare, :valgrind_deps] do
     @ceedling[:test_invoker].setup_and_invoke(
       tests: COLLECTION_ALL_TESTS,
       context: VALGRIND_SYM,
-      options: { :force_run => true, :build_only => false }.merge(TOOL_COLLECTION_TEST_TASKS)
+      options: []
     )
   end
 
-  desc "Run single Valgrind test ([*] test or source file name, no path)."
+  desc "Run single Valgrind test ([*] test or source file name, with optional path)."
   task :* do
     message = "Oops! '#{VALGRIND_ROOT_NAME}:*' isn't a real task. " +
-              "Use a real test or source file name (no path) in place of the wildcard.\n" +
-              "Example: `ceedling #{VALGRIND_ROOT_NAME}:foo.c`"
+              "Use a real test or source file name in place of the wildcard, " +
+              "adding as much of its path as needed to identify one file among any same-named tests.\n" +
+              "Example: `ceedling #{VALGRIND_ROOT_NAME}:foo.c` or `ceedling #{VALGRIND_ROOT_NAME}:unit/foo.c`"
     @ceedling[:loginator].log( message, Verbosity::ERRORS )
   end
 
   desc "Run Valgrind tests by matching regular expression pattern."
-  task :pattern, [:regex] => [:prepare] do |_t, args|
+  task :pattern, [:regex] => [:prepare, :valgrind_deps] do |_t, args|
     matches = []
-    COLLECTION_ALL_TESTS.each { |test| matches << test if test =~ /#{args.regex}/ }
+    # Compile the user's pattern once rather than once per test in the collection
+    pattern = Regexp.new( args.regex )
+    COLLECTION_ALL_TESTS.each { |test| matches << test if test.match?( pattern ) }
     if !matches.empty?
       @ceedling[:test_invoker].setup_and_invoke(
         tests: matches,
         context: VALGRIND_SYM,
-        options: { :force_run => false }.merge(TOOL_COLLECTION_TEST_TASKS)
+        options: []
       )
     else
       @ceedling[:loginator].log( "Found no tests matching pattern /#{args.regex}/", Verbosity::ERRORS )
@@ -54,14 +75,14 @@ namespace VALGRIND_SYM do
   end
 
   desc "Run Valgrind tests whose test path contains [dir] or [dir] substring."
-  task :path, [:dir] => [:prepare] do |_t, args|
+  task :path, [:dir] => [:prepare, :valgrind_deps] do |_t, args|
     matches = []
     COLLECTION_ALL_TESTS.each { |test| matches << test if File.dirname(test).include?(args.dir.tr('\\', '/')) }
     if !matches.empty?
       @ceedling[:test_invoker].setup_and_invoke(
         tests: matches,
         context: VALGRIND_SYM,
-        options: { :force_run => false }.merge(TOOL_COLLECTION_TEST_TASKS)
+        options: []
       )
     else
       @ceedling[:loginator].log( "Found no tests including the given path or path component", Verbosity::ERRORS )
@@ -73,15 +94,16 @@ end
 # Use a rule to handle dynamic per-file tasks: valgrind:foo → build and run foo under Valgrind
 rule(/^#{VALGRIND_TASK_ROOT}\S+$/ => [
   proc do |task_name|
-    test = task_name.strip().sub(/^#{VALGRIND_TASK_ROOT}/, '').chomp(EXTENSION_SOURCE)
-    test = PROJECT_TEST_FILE_PREFIX + test unless test.start_with?(PROJECT_TEST_FILE_PREFIX)
+    # Any directory typed ahead of the name is kept. See TestTaskName.
+    test = TestTaskName.test_name( task_name, root: VALGRIND_TASK_ROOT )
     @ceedling[:file_finder].find_test_file_from_name(test)
   end
 ]) do |test|
   @ceedling[:rake_wrapper][:prepare].invoke
+  @ceedling[:rake_wrapper][:valgrind_deps].invoke
   @ceedling[:test_invoker].setup_and_invoke(
     tests: [test.source],
     context: VALGRIND_SYM,
-    options: { :force_run => true, :build_only => false }.merge(TOOL_COLLECTION_TEST_TASKS)
+    options: []
   )
 end