ceedling 1.1.10 → 1.2.1

data/plugins/sanitize/lib/sanitize_records.rb

@@ -0,0 +1,115 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/constants'
+require 'sanitize_constants'
+require 'sanitize_reports'
+
+# The findings a run's instrumented builds turn up, each with its build context and test.
+# Tests run in parallel, so recording is synchronized. A crashed test's backtrace reruns
+# reproduce the main run's finding and can reach findings it never did, so a finding
+# recorded again is counted once.
+class SanitizeRecords
+
+  def initialize(file_wrapper:, loginator:, reports:)
+    @file_wrapper = file_wrapper
+    @loginator    = loginator
+    @reports      = reports
+    @records      = []
+    @logged_tests = []
+    @written      = []
+    @mutex        = Mutex.new
+  end
+
+  def empty? = @records.empty?
+
+  def self.log_filepath(context, test)
+    return File.join( SANITIZE_ARTIFACTS_PATH, context.to_s, "#{test}.log" )
+  end
+
+  # A crash's explanation, in the keys of the explain_test_crash hook
+  def self.explanation(context, test, finding)
+    return {
+      explanation_heading: 'was stopped by a sanitizer finding',
+      explanation:         SanitizeReports.headline( finding ),
+      explanation_details: ["Boom log: #{log_filepath( context, test )}"]
+    }
+  end
+
+  # Adds a test's new findings to its own log and logs each, except one a crash's notice names
+  def record(context, test, findings, explained: nil)
+    added = @mutex.synchronize { add_new( context, test, findings ) }
+    return if added.empty?
+
+    log = SanitizeRecords.log_filepath( context, test )
+    write_log( log, added )
+    added.each do |finding|
+      next if explained && key( finding ) == key( explained )
+
+      @loginator.log( "#{test}: #{SanitizeReports.headline( finding )} ➡️ see #{log}", Verbosity::ERRORS, LogLabels::ERROR )
+    end
+  end
+
+  def write_reports(context)
+    records   = @records.select { |record| record[:context] == context }
+    directory = File.join( SANITIZE_ARTIFACTS_PATH, context.to_s )
+
+    @reports.each do |report|
+      next if records.empty? && !report.write_empty?
+
+      filepath = File.join( directory, "#{SANITIZE_ROOT_NAME}.#{report.extension}" )
+      @file_wrapper.mkdir( directory )
+      @file_wrapper.write( filepath, report.render( context, records ) )
+      @written << filepath
+    end
+  end
+
+  # Names the reports written, grouped by build context directory
+  def failure_message
+    tests   = @records.map { |record| record[:test] }.uniq.size
+    message = "Sanitizers found #{pluralize( @records.size, 'problem' )} in #{pluralize( tests, 'test' )}"
+    return message if @written.empty?
+
+    return "#{message}. See the reports in #{written_by_directory}."
+  end
+
+  private
+
+  def written_by_directory
+    groups = @written.uniq.group_by { |path| File.dirname( path ) }
+    return groups.map { |dir, paths| "#{dir}: #{paths.map { |path| File.basename( path ) }.join( ', ' )}" }.join( '; ' )
+  end
+
+  # The findings not yet recorded for this test, now recorded
+  def add_new(context, test, findings)
+    added = findings.reject { |finding| @records.any? { |record| same?( record, context, test, finding ) } }.uniq { |finding| key( finding ) }
+    added.each { |finding| @records << { context: context, test: test, finding: finding } }
+    return added
+  end
+
+  def same?(record, context, test, finding)
+    return record[:context] == context && record[:test] == test && key( record[:finding] ) == key( finding )
+  end
+
+  def key(finding)
+    return [finding.sanitizer, finding.kind, finding.file, finding.line]
+  end
+
+  # A build's first report for a test replaces the last build's log, and later ones follow it
+  def write_log(log, findings)
+    first = @mutex.synchronize { !@logged_tests.include?( log ) && @logged_tests << log }
+    text  = findings.map( &:text ).join( "\n" )
+
+    @file_wrapper.mkdir( File.dirname( log ) )
+    @file_wrapper.write( log, first ? text : "\n#{text}", first ? 'w' : 'a' )
+  end
+
+  def pluralize(count, word)
+    return "#{count} #{word}#{count == 1 ? '' : 's'}"
+  end
+
+end

data/plugins/sanitize/lib/sanitize_reports.rb

@@ -0,0 +1,135 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'json'
+require 'ceedling/exceptions'
+
+# The reports a build context's sanitizer findings can be written as. Each renders a
+# context's finding records, each a test and its finding, as a file's contents and names
+# that file's extension. The text log is the default; JSON and SARIF are structured
+# reports for other tools to read.
+module SanitizeReports
+
+  LABELS = { address: 'AddressSanitizer', undefined: 'UndefinedBehaviorSanitizer', leak: 'LeakSanitizer' }.freeze
+
+  # A finding's sanitizer, kind, and location in one line. UBSan names every finding
+  # undefined behavior, so its message says what happened instead.
+  def self.headline(finding)
+    location = finding.file ? " at #{finding.file}:#{finding.line}" : ''
+    what     = finding.sanitizer == :undefined ? finding.message : finding.kind
+    return "#{LABELS[finding.sanitizer]} #{what}#{location}"
+  end
+
+  # Each report as the sanitizer printed it, under its test. A context without findings has
+  # no log.
+  class Text
+    def extension = 'log'
+    def write_empty? = false
+
+    def render(_context, records)
+      return records.map { |record| "#{record[:test]}: #{SanitizeReports.headline( record[:finding] )}\n#{record[:finding].text}\n" }.join
+    end
+  end
+
+  # Ceedling's own JSON layout: a context and its findings' parsed fields. A context without
+  # findings is still written, saying so.
+  class Json
+    def extension = 'json'
+    def write_empty? = true
+
+    def render(context, records)
+      findings = records.map { |record| { test: record[:test] }.merge( record[:finding].to_h.except( :text ) ) }
+      return JSON.pretty_generate( { context: context.to_s, findings: findings } ) + "\n"
+    end
+  end
+
+  # SARIF 2.1.0, the OASIS standard for analysis results, as one run by Ceedling per build
+  # context. Every finding is an error. A context without findings is a run with no results,
+  # which tells a code scanning service that earlier results are gone.
+  class Sarif
+    SCHEMA = 'https://json.schemastore.org/sarif-2.1.0.json'.freeze
+
+    def initialize(version)
+      @version = version
+    end
+
+    def extension = 'sarif'
+    def write_empty? = true
+
+    def render(context, records)
+      rules = records.map { |record| rule_id( record[:finding] ) }.uniq.map { |id| { id: id } }
+      run   = {
+        tool:              { driver: { name: 'Ceedling', version: @version, informationUri: 'https://www.throwtheswitch.org/ceedling', rules: rules } },
+        automationDetails: { id: "#{context}/" },
+        results:           records.map { |record| result( record ) }
+      }
+      return JSON.pretty_generate( { '$schema': SCHEMA, version: '2.1.0', runs: [run] } ) + "\n"
+    end
+
+    private
+
+    def rule_id(finding)
+      return "#{finding.sanitizer}/#{finding.kind}"
+    end
+
+    def result(record)
+      finding = record[:finding]
+      result  = { ruleId: rule_id( finding ), level: 'error', message: { text: "#{record[:test]}: #{finding.message}" } }
+      result[:locations]  = [location( finding )] if finding.file
+      result[:properties] = { test: record[:test], text: finding.text }
+      return result
+    end
+
+    def location(finding)
+      region = { startLine: finding.line }
+      region[:startColumn] = finding.column if finding.column
+      return { physicalLocation: { artifactLocation: artifact( finding.file ), region: region } }
+    end
+
+    # A relative path is relative to the project, which SARIF names %SRCROOT%, and loses any
+    # leading `./` that would keep it from matching the project's file. An absolute path,
+    # POSIX or Windows, is a file URI.
+    def artifact(file)
+      path = file.tr( '\\', '/' ).sub( %r{\A(?:\./)+}, '' )
+      return { uri: "file://#{encode( path )}" } if path.start_with?( '/' )
+      return { uri: "file:///#{encode( path )}" } if path.match?( %r{\A[A-Za-z]:/} )
+
+      return { uri: encode( path ), uriBaseId: '%SRCROOT%' }
+    end
+
+    def encode(path)
+      return path.gsub( %r{[^A-Za-z0-9\-._~/:]} ) { |char| char.bytes.map { |byte| format( '%%%02X', byte ) }.join }
+    end
+  end
+
+  NAMES = %i[text json sarif].freeze
+
+  # The configured reports, each once, in the order given. Entries are symbols or strings in
+  # any letter case, and none configured is the text log alone.
+  def self.load(entries, version:)
+    names = Array( entries ).map { |entry| name( entry ) }.uniq
+    names = [:text] if names.empty?
+
+    return names.map do |name|
+      case name
+      when :text  then Text.new
+      when :json  then Json.new
+      when :sarif then Sarif.new( version )
+      end
+    end
+  end
+
+  def self.name(entry)
+    report = entry.to_s.downcase.to_sym
+    return report if (entry.is_a?( String ) || entry.is_a?( Symbol )) && NAMES.include?( report )
+
+    choices = NAMES.map( &:inspect ).join( ', ' )
+    raise CeedlingException.new( ":sanitize ↳ :reports entry `#{entry}` is not a report. Choose from #{choices}." )
+  end
+  private_class_method :name
+
+end

data/plugins/sanitize/sanitize.rake

@@ -0,0 +1,108 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+##
+## This Rakefile plugin builds and runs test executables with sanitizers via its
+## build hooks. It calls the test invoker directly (same as gcov.rake)
+## to avoid marking test: tasks as invoked.
+##
+
+# The plugin's own build context, kept apart from test builds
+CLEAN.include(File.join(SANITIZE_BUILD_PATH, BUILD_OUT_DIR, '*'))
+CLEAN.include(File.join(SANITIZE_BUILD_PATH, BUILD_RESULTS_DIR, '*'))
+CLEAN.include(File.join(SANITIZE_BUILD_PATH, BUILD_DEPENDENCIES_DIR, '*'))
+CLEAN.include(File.join(SANITIZE_ARTIFACTS_PATH, '**/*'))
+
+CLOBBER.include(File.join(SANITIZE_BUILD_PATH, '**/*'))
+
+# The artifacts path as a Rake directory task, so the `directories` prerequisite below
+# can resolve it
+directory(SANITIZE_ARTIFACTS_PATH)
+
+task directories: [SANITIZE_ARTIFACTS_PATH]
+
+# Every sanitize: task depends on this. It is the one place the host platform and
+# the fixture tool are checked, so a build that never runs a sanitize: task
+# never requires them. See Sanitize#validate_environment!.
+task :sanitize_deps do
+  @ceedling[SANITIZE_SYM].validate_environment!
+end
+
+task :sanitize => [:prepare, :sanitize_deps] do
+  Rake.application['sanitize:all'].invoke
+end
+
+namespace SANITIZE_SYM do
+
+  desc "Run all unit tests with sanitizers (also just 'sanitize' works)."
+  task :all => [:prepare, :sanitize_deps] do
+    @ceedling[:test_invoker].setup_and_invoke(
+      tests: COLLECTION_ALL_TESTS,
+      context: SANITIZE_SYM,
+      options: []
+    )
+  end
+
+  desc "Run single sanitized test ([*] test or source file name, with optional path)."
+  task :* do
+    message = "Oops! '#{SANITIZE_ROOT_NAME}:*' isn't a real task. " +
+              "Use a real test or source file name in place of the wildcard, " +
+              "adding as much of its path as needed to identify one file among any same-named tests.\n" +
+              "Example: `ceedling #{SANITIZE_ROOT_NAME}:foo.c` or `ceedling #{SANITIZE_ROOT_NAME}:unit/foo.c`"
+    @ceedling[:loginator].log( message, Verbosity::ERRORS )
+  end
+
+  desc "Run sanitized tests by matching regular expression pattern."
+  task :pattern, [:regex] => [:prepare, :sanitize_deps] do |_t, args|
+    matches = []
+    # Compile the user's pattern once rather than once per test in the collection
+    pattern = Regexp.new( args.regex )
+    COLLECTION_ALL_TESTS.each { |test| matches << test if test.match?( pattern ) }
+    if !matches.empty?
+      @ceedling[:test_invoker].setup_and_invoke(
+        tests: matches,
+        context: SANITIZE_SYM,
+        options: []
+      )
+    else
+      @ceedling[:loginator].log( "Found no tests matching pattern /#{args.regex}/", Verbosity::ERRORS )
+    end
+  end
+
+  desc "Run sanitized tests whose test path contains [dir] or [dir] substring."
+  task :path, [:dir] => [:prepare, :sanitize_deps] do |_t, args|
+    matches = []
+    COLLECTION_ALL_TESTS.each { |test| matches << test if File.dirname(test).include?(args.dir.tr('\\', '/')) }
+    if !matches.empty?
+      @ceedling[:test_invoker].setup_and_invoke(
+        tests: matches,
+        context: SANITIZE_SYM,
+        options: []
+      )
+    else
+      @ceedling[:loginator].log( "Found no tests including the given path or path component", Verbosity::ERRORS )
+    end
+  end
+
+end
+
+# Use a rule to handle dynamic per-file tasks: sanitize:foo → build and run foo with sanitizers
+rule(/^#{SANITIZE_TASK_ROOT}\S+$/ => [
+  proc do |task_name|
+    # Any directory typed ahead of the name is kept. See TestTaskName.
+    test = TestTaskName.test_name( task_name, root: SANITIZE_TASK_ROOT )
+    @ceedling[:file_finder].find_test_file_from_name(test)
+  end
+]) do |test|
+  @ceedling[:rake_wrapper][:prepare].invoke
+  @ceedling[:rake_wrapper][:sanitize_deps].invoke
+  @ceedling[:test_invoker].setup_and_invoke(
+    tests: [test.source],
+    context: SANITIZE_SYM,
+    options: []
+  )
+end

data/plugins/valgrind/config/defaults.yml

@@ -10,7 +10,13 @@
   # Register a build failure if valgrind reports memory errors in any test binary
   :fail_build: TRUE
 
-  # Valgrind flags (do not include --log-file or the ${1} executable token)
+  # Write a machine-readable XML report per test binary alongside the text log
+  :xml_report: FALSE
+
+  # Valgrind suppression files, one --suppressions= argument each
+  :suppressions: []
+
+  # Valgrind flags (do not include --log-file, --xml-file, --suppressions, or the ${1} executable token)
   :arguments:
     - "--leak-check=full"
     - "--show-leak-kinds=all"

data/plugins/valgrind/config/defaults_valgrind.rb

@@ -6,10 +6,15 @@
 # =========================================================================
 
 
+# `:optional => true` keeps Ceedling's project-wide tool validation from failing a
+# build merely because this plugin is enabled. Valgrind is an external tool that runs
+# on Linux and other Unix-like systems only, so an enabled plugin is not evidence the
+# executable exists. The plugin validates it for real at `valgrind:` task time instead.
+# See Valgrind#validate_environment!.
 DEFAULT_VALGRIND_TOOL = {
     :executable => FilePathUtils.os_executable_ext('valgrind').freeze,
     :name => 'default_valgrind'.freeze,
-    :optional => false.freeze,
+    :optional => true.freeze,
     :arguments => [].freeze  # Arguments built dynamically by the plugin from :valgrind: :arguments: config
     }