ceedling 1.1.10 → 1.2.0

data/spec/support/system/system_context.rb

@@ -8,20 +8,62 @@
 require 'fileutils'
 require 'bundler'
 require 'ceedling/encodinator'
+require 'ceedling/file_wrapper'
+require 'ceedling/verbosinator'
+require 'ceedling/yaml_wrapper'
 require_relative 'gem_dir_layout'
 
 class SystemContext
   class VerificationFailed < RuntimeError; end
-  class InvalidBackupEnv < RuntimeError; end
 
   attr_reader :dir, :gem, :console_summary, :raw_output, :last_exit_status, :last_cmd
 
   SYSTEM_TEST_KEEP_ENV = 'CEEDLING_SYSTEM_TEST_KEEP'
 
+  # Dynamic analysis of Ceedling's own generated C -- runners, mocks, Partials.
+  #
+  # Names a sanitizer flavor, which is a file in SANITIZERS_DIR, so adding a
+  # flavor needs no change here. Follows CEEDLING_TEST_COVERAGE's convention of a
+  # named value rather than a boolean, so a second kind of instrumentation (leak
+  # detection) is another value rather than another variable.
+  SANITIZERS_ENV = 'CEEDLING_TEST_SANITIZERS'
+
+  # Slot 9, not 1: spec/system/mixin_ordering_spec.rb sets CEEDLING_MIXIN_1
+  # itself and asserts the resulting merge, so sharing that slot would collide
+  # with a spec. Higher also means merging last among env mixins.
+  SANITIZERS_MIXIN_ENV = 'CEEDLING_MIXIN_9'
+
+  REPO_ROOT = File.expand_path( File.join( File.dirname(__FILE__), '..', '..', '..' ) )
+
+  SANITIZERS_DIR = File.join( REPO_ROOT, 'spec', 'support', 'system', 'sanitizers' )
+
+  # Set per example by the "cannot be sanitized" shared context (see
+  # spec_system_helper.rb). Class-level rather than instance state because specs
+  # build their own SystemContext in before(:all) while this toggles in
+  # before/after(:each) -- a class-level flag makes the two order-independent.
+  @@sanitizers_disabled = false
+
+  # Flavor file path => its :sanitizer_runtime hash. See #sanitizer_runtime.
+  @@sanitizer_runtime = {}
+
+  def self.sanitizers_disabled=(disabled)
+    @@sanitizers_disabled = disabled
+  end
+
+  def self.sanitizers_disabled?
+    @@sanitizers_disabled
+  end
+
+  # Root for retained system test artifacts, relative to the working directory.
+  # Named once here because the log writer in spec_system_helper.rb, the gemspec,
+  # .rubocop.yml, .gitignore, and CI all reach this path by name.
+  ARTIFACTS_DIR = 'specout'
+
   # Shared gem installation — built once by setup_shared_gem!, reused by every deploy_gem call.
   # Eliminates redundant `bundle install` runs (one per describe group → one per suite).
   @@shared_gem_dir = nil
   @@shared_gem     = nil
+  @@git_repo       = nil
 
   def self.setup_shared_gem!
     return if @@shared_gem_dir
@@ -30,34 +72,41 @@ class SystemContext
     shared_gem = GemDirLayout.new(shared_dir)
 
     git_repo = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..'))
-    File.write(
-      File.join(shared_dir, 'Gemfile'),
-      [
-        %Q{source "http://rubygems.org/"},
-        %Q{gem "rake"},
-        %Q{gem "constructor"},
-        %Q{gem "diy"},
-        %Q{gem "thor"},
-        %Q{gem "deep_merge"},
-        %Q{gem "unicode-display_width"},
-        %Q{gem "ceedling", :path => '#{git_repo}'}
-      ].join("\n")
-    )
+    gemfile_lines = [
+      %Q{source "http://rubygems.org/"},
+      %Q{gem "rake"},
+      %Q{gem "constructor"},
+      # `diy` is absent deliberately, mirroring ceedling.gemspec. Its source is
+      # vendored and reaches the child process through the gem's own require_paths.
+      %Q{gem "thor"},
+      %Q{gem "deep_merge"},
+      %Q{gem "unicode-display_width"},
+      %Q{gem "ceedling", :path => '#{git_repo}'}
+    ]
+    # This machine-generated Gemfile is never committed -- adding simplecov here only
+    # when coverage mode is on keeps every other run's deployed environment identical
+    # to what a real installed gem's own dependencies actually are. Pinned to the same
+    # constraint as the main Gemfile's Ruby->=3.2 branch so both processes run the
+    # identical SimpleCov version -- an unpinned resolve here could otherwise drift to
+    # whatever's newest on rubygems.org independent of the main Gemfile.lock. No
+    # RUBY_VERSION branch needed here (unlike the main Gemfile): this line only ever
+    # runs at all when CEEDLING_TEST_COVERAGE == 'system', itself only ever set on the
+    # Ruby 3.3 CI leg.
+    gemfile_lines << %Q{gem "simplecov", "~> 1.1"} if ENV['CEEDLING_TEST_COVERAGE'] == 'system'
+    File.write( File.join(shared_dir, 'Gemfile'), gemfile_lines.join("\n") )
 
     Dir.chdir(shared_dir) do
       begin
-        # Bundler.with_unbundled_env restores the pre-`bundle exec` environment for this
-        # block -- clearing not just BUNDLE_GEMFILE/BUNDLE_BIN_PATH/RUBYOPT (the previous,
-        # incomplete hand-rolled list here) but also GEM_HOME/GEM_PATH/RUBYLIB/PATH and every
-        # other var Bundler sets when this spec suite itself runs under `bundle exec` against
-        # a non-default GEM_HOME (e.g. `bundle install --path vendor/bundle`). Any one of
-        # those left in place is enough on its own for the `bundle exec ruby -S ceedling
-        # version` verification below to load the *outer* project's bundler/gems instead of
-        # this shared gem's own `--local path` install, and fail to find this Gemfile's exact
-        # pinned versions there.
+        # Bundler.with_unbundled_env restores the pre-`bundle exec` environment for
+        # this block -- clearing not just BUNDLE_GEMFILE/BUNDLE_BIN_PATH/RUBYOPT but
+        # also GEM_HOME/GEM_PATH/RUBYLIB and every other var Bundler sets when this
+        # spec suite itself runs under `bundle exec` against a non-default GEM_HOME
+        # (e.g. `bundle install --path vendor/bundle`). Without it, this nested
+        # install inherits the outer Bundler's env and fails to find its own
+        # dependencies under a completely different Gemfile.
         deploy_output = Bundler.with_unbundled_env do
           output  = `bundle config set --local path '#{shared_gem.install_dir}' 2>&1`
-          # --prefer-local: Without it, Bundler resolves gems (e.g. `erb`) fresh from
+          # --prefer-local: Without it, Bundler resolves gems (e.g. `benchmark`) fresh from
           # rubygems repository even when Ruby's default-gem copy satisfies the Gemfile constraint.
           output += `bundle install --prefer-local 2>&1`
           output
@@ -78,20 +127,23 @@ class SystemContext
 
     @@shared_gem_dir = shared_dir
     @@shared_gem     = shared_gem
+    @@git_repo       = git_repo
   end
 
   def self.cleanup_shared_gem!
     FileUtils.rm_rf(@@shared_gem_dir) if @@shared_gem_dir
     @@shared_gem_dir = nil
     @@shared_gem     = nil
+    @@git_repo       = nil
   end
 
   def initialize
     if ENV[SYSTEM_TEST_KEEP_ENV]
-      # In either debug mode ('failures' or 'all'), root the temp dir inside systests/proj/
-      # so that done! can rename it to pass/ or fail/ on the same filesystem without a
-      # cross-device copy. The specific subdir (pass/ or fail/) is determined by done!.
-      base = File.join(Dir.pwd, 'systests', 'proj')
+      # In either debug mode ('failures' or 'all'), root the temp dir inside the
+      # artifacts directory's proj/ so that done! can rename it to pass/ or fail/ on
+      # the same filesystem without a cross-device copy. The specific subdir (pass/
+      # or fail/) is determined by done!.
+      base = File.join(Dir.pwd, ARTIFACTS_DIR, 'proj')
       FileUtils.mkdir_p(base)
       @dir = Dir.mktmpdir(nil, base)
     else
@@ -140,18 +192,40 @@ class SystemContext
   end
 
   def with_context
-    Dir.chdir @dir do |current_dir|
+    # Resolved out here, before with_constrained_env: Bundler.with_unbundled_env
+    # restores ENV from the snapshot it took when Bundler was first loaded, so
+    # CEEDLING_TEST_SANITIZERS is not readable inside that block unless it
+    # happened to be set before this process started. Resolving first makes this
+    # work whoever sets the variable, and whenever.
+    sanitizers = sanitizer_environment
+
+    Dir.chdir @dir do |_current_dir|
       with_constrained_env do
         # Point bundle exec to the shared Gemfile so it works from any project directory.
-        # constrain_env removes BUNDLE_GEMFILE; we re-set it here within the constrained scope.
+        # with_constrained_env's unbundled environment has no BUNDLE_GEMFILE of its
+        # own; set it here so the yielded commands resolve against the shared gem.
         ENV['BUNDLE_GEMFILE'] = File.join(@@shared_gem_dir, 'Gemfile') if @@shared_gem_dir
         ENV['RUBYLIB'] = @gem.lib
         ENV['RUBYPATH'] = @gem.bin
 
+        # RUBYOPT survives Bundler's env-stripping above only because it's set here,
+        # inside this block, rather than by whatever called into this method -- forces
+        # spec/support/system/simplecov_boot.rb to load before the child process's own
+        # requires run, the same way a coverage tool would hook any externally invoked
+        # CLI. CEEDLING_TEST_COVERAGE_ROOT tells that boot script where this repo (and
+        # its shared .simplecov config) actually live, since the child's own CWD is
+        # this ephemeral deployed project directory, not the repo.
+        if ENV['CEEDLING_TEST_COVERAGE'] == 'system'
+          ENV['RUBYOPT'] = "-r#{File.join(File.dirname(__FILE__), 'simplecov_boot.rb')}"
+          ENV['CEEDLING_TEST_COVERAGE_ROOT'] = @@git_repo
+        end
+
         ENV['LANG'] = 'en_US.UTF-8'
         ENV['LANGUAGE'] = 'en_US.UTF-8'
         ENV['LC_ALL'] = 'en_US.UTF-8'
 
+        sanitizers.each { |name, value| ENV[name] = value }
+
         yield
       end
     end
@@ -159,63 +233,111 @@ class SystemContext
 
   ############################################################
   # Functions for manipulating environment settings during tests:
-  def backup_env
-    @_env = ENV.to_hash
-  end
 
-  def reduce_env(destroy_keys=[])
-    ENV.keys.each {|k| ENV.delete(k) if destroy_keys.include?(k) }
-  end
+  # Translates CEEDLING_TEST_SANITIZERS into the environment variables Ceedling
+  # and the sanitizer runtime actually consume, as a hash for with_context to
+  # export. Empty when no flavor is selected or the current spec has opted out,
+  # which is what leaves every other matrix leg untouched.
+  #
+  # Nothing is needed downstream to make a finding fail a build:
+  # GeneratorHelper#test_crash? already reads both shapes a sanitizer produces --
+  # SIGABRT, and "zero failures reported but a nonzero unsignaled exit" -- as a
+  # crashed test executable.
+  def sanitizer_environment
+    return {} if self.class.sanitizers_disabled?
+
+    flavor = ENV[SANITIZERS_ENV].to_s.strip
+    return {} if flavor.empty?
+
+    path = File.join( SANITIZERS_DIR, "#{flavor}.yml" )
+    unless File.exist?( path )
+      available = Dir[File.join( SANITIZERS_DIR, '*.yml' )].map { |f| File.basename( f, '.yml' ) }.sort
+      raise VerificationFailed,
+        "#{SANITIZERS_ENV} names no sanitizer flavor: '#{flavor}'. " \
+        "Available flavors: #{available.join(', ')}. " \
+        "Failing here rather than running uninstrumented -- a silent fallback would report " \
+        "success while testing none of what it claims to."
+    end
 
-  def constrain_env
-    destroy_keys = %w{BUNDLE_GEMFILE BUNDLE_BIN_PATH RUBYOPT}
-    reduce_env(destroy_keys)
+    # The mixin path is absolute because each spec chdir's into its own ephemeral
+    # project directory and Ceedling resolves a mixin path against that CWD.
+    #
+    # Each flavor owns its runtime options -- they are the only difference
+    # between asan_ubsan and asan_ubsan_leaks -- and they reach the test
+    # executable because it inherits this environment through Ceedling.
+    environment = { SANITIZERS_MIXIN_ENV => path }
+    sanitizer_runtime( path ).each { |name, value| environment[name.to_s] = value.to_s }
+    environment
   end
 
-  def restore_env
-    if @_env
-      # delete environment variables we've added since we started
-      ENV.to_hash.each_pair {|k,v| ENV.delete(k) unless @_env.include?(k) }
-
-      # restore environment variables we've modified since we started
-      @_env.each_pair {|k,v| ENV[k] = v}
-    else
-      raise InvalidBackupEnv.new
+  # Memoized per flavor file: with_context runs on every build in the suite, and
+  # re-reading the same YAML thousands of times buys nothing.
+  def sanitizer_runtime(path)
+    @@sanitizer_runtime[path] ||= begin
+      file_wrapper = FileWrapper.new({
+        :loginator    => NullLoginator.new,
+        :verbosinator => Verbosinator.new
+      })
+      hash = YamlWrapper.new({ file_wrapper: file_wrapper }).load( path )
+      hash[:sanitizer_runtime] || {}
     end
   end
 
+  # Runs the block against the pre-`bundle exec` environment (see the matching
+  # comment on Bundler.with_unbundled_env in setup_shared_gem! above) so a
+  # subprocess spawned inside -- e.g. `bundle exec ruby -S ceedling ...` against
+  # the shared gem's own Gemfile in with_context below -- resolves against that
+  # Gemfile rather than whatever Bundler context this spec suite itself is
+  # already running under.
   def with_constrained_env
-    begin
-      backup_env
-      constrain_env
-      yield
-    ensure
-      restore_env
-    end
+    Bundler.with_unbundled_env { yield }
   end
 
   ############################################################
   # Functions for manipulating project.yml files during tests:
+
+  # `self` here is a plain SystemContext instance, not an RSpec example, so
+  # RSpec::Mocks' `double` isn't reachable -- a trivial no-op stands in for it.
+  class NullLoginator
+    def log(*); end
+  end
+
   def merge_project_yml_for_test(settings, show_final=false)
-    yaml_wrapper = YamlWrapper.new
+    file_wrapper = FileWrapper.new({
+      :loginator    => NullLoginator.new,
+      :verbosinator => Verbosinator.new
+    })
+    yaml_wrapper = YamlWrapper.new({ file_wrapper: file_wrapper })
     project_hash = yaml_wrapper.load('project.yml')
     project_hash.deep_merge!(settings)
     puts "\n\n#{project_hash.to_yaml}\n\n" if show_final
     yaml_wrapper.dump('project.yml', project_hash)
   end
 
+  # Deletes one setting, e.g. [:extension, :executable], so Ceedling's default applies
+  def remove_project_yml_option_for_test(*keys)
+    file_wrapper = FileWrapper.new({
+      :loginator    => NullLoginator.new,
+      :verbosinator => Verbosinator.new
+    })
+    yaml_wrapper = YamlWrapper.new({ file_wrapper: file_wrapper })
+    project_hash = yaml_wrapper.load('project.yml')
+    project_hash.dig( *keys[0...-1] )&.delete( keys.last )
+    yaml_wrapper.dump('project.yml', project_hash)
+  end
+
   def append_project_yml_for_test(new_args)
-    fake_prj_yml= "#{File.read('project.yml')}\n#{new_args}"
+    fake_prj_yml= "#{read_spec_file('project.yml')}\n#{new_args}"
     File.write('project.yml', fake_prj_yml, mode: 'w')
   end
 
   def uncomment_project_yml_option_for_test(option)
-    fake_prj_yml= File.read('project.yml').gsub(/\##{option}/,option)
+    fake_prj_yml= read_spec_file('project.yml').gsub(/\##{option}/,option)
     File.write('project.yml', fake_prj_yml, mode: 'w')
   end
 
   def comment_project_yml_option_for_test(option)
-    fake_prj_yml= File.read('project.yml').gsub(/#{option}/,"##{option}")
+    fake_prj_yml= read_spec_file('project.yml').gsub(/#{option}/,"##{option}")
     File.write('project.yml', fake_prj_yml, mode: 'w')
   end
 

data/spec/support/template_render_cases.rb

@@ -0,0 +1,166 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/constants'
+require 'ceedling/defaults'
+
+# The report templates and the render inputs behind the golden fixtures in
+# spec/support/fixtures/templates/.
+#
+# Shared so the generator that produced those fixtures and the spec that checks
+# against them read from one definition. Two copies would drift, and a drifted
+# case would silently stop comparing what it claims to compare.
+#
+# Cases are chosen to reach every branch across the three templates: empty runs,
+# passing runs, failures and ignores with empty, single line, and multi line
+# messages, stdout capture, a non-test context that prepends a header, and each
+# shape of missing coverage data.
+module TemplateRenderCases
+
+  ROOT = File.expand_path( '../../..', __FILE__ )
+
+  FIXTURES_PATH = File.join( ROOT, 'spec', 'support', 'fixtures', 'templates' )
+
+  # Name => template string. The ide template is Ceedling's default results template,
+  # a constant rather than an asset file, which is why it is read differently from the
+  # other two. The IDE plugin's own template, set to :legacy, must match the same ide
+  # fixtures, as its spec checks.
+  #
+  # Every template is normalized to line feeds. The fixtures record what ERB
+  # produced from the canonical template content, so the comparison has to be about
+  # rendering rather than about how a checkout happened to store the templates.
+  # Git converts line endings per .gitattributes, Ruby's text mode converts again
+  # when reading on Windows, and the ide template arrives from a Ruby source file
+  # subject to both. Without this the same rendering logic would be measured
+  # against three different inputs depending on platform and git settings.
+  #
+  # Rendering of carriage returns is not skipped by doing this. It is pinned
+  # directly in templateinator_spec.rb, by an example whose template is an inline
+  # string and therefore immune to any of the above.
+  def self.templates
+    return {
+      'pretty'    => read_template( 'plugins', 'report_tests_pretty_stdout', 'assets', 'test_results.template' ),
+      'bullseye'  => read_template( 'plugins', 'bullseye', 'assets', 'coverage.template' ),
+      'ide'       => normalize_newlines( DEFAULT_TESTS_RESULTS_REPORT_TEMPLATE ),
+    }
+  end
+
+  # Read as bytes, then labeled UTF-8 rather than left at whatever the platform's
+  # default external encoding happens to be. Templates are UTF-8 in the repo, and a
+  # template string carrying a different label would fail to concatenate with the
+  # compiler's own literals the moment one held a non-ASCII character.
+  def self.read_template(*path_parts)
+    bytes = File.binread( File.join( ROOT, *path_parts ) )
+    return normalize_newlines( bytes.force_encoding( Encoding::UTF_8 ) )
+  end
+
+  def self.normalize_newlines(template)
+    return template.gsub( "\r\n", "\n" )
+  end
+
+  MULTILINE_MESSAGE = "Expected 1 Was 2.\nsecond line\nthird line"
+
+  # Every case carries both :results and :coverage so all three templates can render
+  # all of them. The templates each ignore what they do not use.
+  def self.cases
+    return {
+      'empty'                   => build( total: 0 ),
+
+      'all_pass'                => build( passed: 2 ),
+
+      'one_failure_matching'    => build( passed: 1, failures: [ failure( 'Expected 1 Was 2.' ) ] ),
+
+      'one_failure_nonmatching' => build( passed: 1, failures: [ failure( 'Something else entirely' ) ] ),
+
+      'one_failure_multiline'   => build( passed: 1, failures: [ failure( MULTILINE_MESSAGE ) ] ),
+
+      'one_failure_no_message'  => build( passed: 1, failures: [ failure( '' ) ] ),
+
+      'one_ignore_with_message' => build( passed: 1, ignores: [ ignore( 'not ready yet' ) ] ),
+
+      'one_ignore_no_message'   => build( passed: 1, ignores: [ ignore( '' ) ] ),
+
+      'stdout_only'             => build( passed: 1, stdout: [ stdout_entry ] ),
+
+      'mixed_all'               => build(
+                                     passed: 2,
+                                     failures: [ failure( MULTILINE_MESSAGE ), failure( 'Expected 3 Was 4.', file: 'test/TestBar.c' ) ],
+                                     ignores:  [ ignore( 'skipped' ) ],
+                                     stdout:   [ stdout_entry ]
+                                   ),
+
+      # A non-test context prepends a header to every banner.
+      'gcov_context'            => build(
+                                     context: :gcov,
+                                     passed: 1,
+                                     failures: [ failure( MULTILINE_MESSAGE ) ],
+                                     ignores:  [ ignore( 'skipped' ) ],
+                                     stdout:   [ stdout_entry ]
+                                   ),
+
+      'coverage_functions_only' => build( passed: 1, coverage: { functions: 91, branches: nil } ),
+
+      'coverage_branches_only'  => build( passed: 1, coverage: { functions: nil, branches: 7 } ),
+
+      'coverage_none'           => build( passed: 1, coverage: { functions: nil, branches: nil } ),
+    }
+  end
+
+  ### Private ###
+
+  def self.failure(message, file: 'test/TestFoo.c')
+    return {
+      source: { file: file },
+      collection: [ { test: 'test_fails', line: 30, message: message } ]
+    }
+  end
+
+  def self.ignore(message, file: 'test/TestFoo.c')
+    return {
+      source: { file: file },
+      collection: [ { test: 'test_ignored', line: 40, message: message } ]
+    }
+  end
+
+  def self.stdout_entry(file: 'test/TestFoo.c')
+    return {
+      source: { file: file },
+      collection: [ 'debug line one', 'debug line two' ]
+    }
+  end
+
+  def self.build(context: TEST_SYM, total: nil, passed: 0, failures: [], ignores: [], stdout: [],
+                 coverage: { functions: 91, branches: 72 })
+    failed  = failures.sum { |entry| entry[:collection].length }
+    ignored = ignores.sum  { |entry| entry[:collection].length }
+
+    successes = passed.zero? ? [] : [ {
+      source: { file: 'test/TestFoo.c' },
+      collection: (1..passed).map { |i| { test: "test_passes_#{i}", line: i, message: '' } }
+    } ]
+
+    return {
+      context: context,
+      coverage: coverage,
+      results: {
+        counts: {
+          total:   total.nil? ? (passed + failed + ignored) : total,
+          passed:  passed,
+          failed:  failed,
+          ignored: ignored,
+          stdout:  stdout.sum { |entry| entry[:collection].length }
+        },
+        successes: successes,
+        failures:  failures,
+        ignores:   ignores,
+        stdout:    stdout,
+        times:     { 'test/TestFoo.c' => 0.0123, 'test/TestBar.c' => 0.5 }
+      }
+    }
+  end
+
+end