ceedling 0.31.0 → 1.0.0

data/spec/spec_system_helper.rb

@@ -1,21 +1,15 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
 require 'fileutils'
 require 'tmpdir'
 require 'ceedling/yaml_wrapper'
 require 'spec_helper'
-
-if Gem.ruby_version >= Gem::Version.new("2.5.0")
-  Modulegenerator = Struct.new(:project_root, :source_root, :inc_root, :test_root, keyword_init: true) do
-    def initialize(project_root: "./", source_root: "src/", inc_root: "src/", test_root: "test/")
-      super
-    end
-  end
-else
-  Modulegenerator = Struct.new(:project_root, :source_root, :inc_root, :test_root) do
-    def initialize(project_root: "./", source_root: "src/", inc_root: "src/", test_root: "test/")
-      super(project_root, source_root, inc_root, test_root)
-    end
-  end
-end
+require 'deep_merge'
 
 def test_asset_path(asset_file_name)
   File.join(File.dirname(__FILE__), '..', 'assets', asset_file_name)
@@ -29,13 +23,6 @@ def convert_slashes(path)
   end
 end
 
-def add_project_settings(project_file_path, settings)
-  yaml_wrapper = YamlWrapper.new
-  project_hash = yaml_wrapper.load(project_file_path)
-  project_hash.deep_merge(settings)
-  yaml_wrapper.dump(project_file_path, project_hash)
-end
-
 class GemDirLayout
   attr_reader :gem_dir_base_name
 
@@ -51,8 +38,8 @@ class GemDirLayout
 end
 
 class SystemContext
-  class VerificationFailed < Exception; end
-  class InvalidBackupEnv < Exception; end
+  class VerificationFailed < RuntimeError; end
+  class InvalidBackupEnv < RuntimeError; end
 
   attr_reader :dir, :gem
 
@@ -66,11 +53,17 @@ class SystemContext
   end
 
   def deploy_gem
-    git_repo = File.expand_path(File.join(File.dirname(__FILE__), '..'))
-    bundler_gem_file_data = [ %Q{source "http://rubygems.org/"},
-                              %Q{gem "rake"},
-                              %Q{gem "ceedling", :path => '#{git_repo.to_s}'}
-                            ].join("\n")
+    git_repo = File.expand_path( File.join( File.dirname( __FILE__ ), '..') )
+    bundler_gem_file_data = [
+      %Q{source "http://rubygems.org/"},
+      %Q{gem "rake"},
+      %Q{gem "constructor"},
+      %Q{gem "diy"},
+      %Q{gem "thor"},
+      %Q{gem "deep_merge"},
+      %Q{gem "unicode-display_width"},
+      %Q{gem "ceedling", :path => '#{git_repo}'}
+    ].join("\n")
 
     File.open(File.join(@dir, "Gemfile"), "w+") do |f|
       f.write(bundler_gem_file_data)
@@ -87,6 +80,7 @@ class SystemContext
         end
       end
     end
+
   end
 
   # Does a few things:
@@ -105,14 +99,19 @@ class SystemContext
         ENV['RUBYLIB'] = @gem.lib
         ENV['RUBYPATH'] = @gem.bin
 
+        ENV['LANG'] = 'en_US.UTF-8'
+        ENV['LANGUAGE'] = 'en_US.UTF-8'
+        ENV['LC_ALL'] = 'en_US.UTF-8'
+
         yield
       end
     end
   end
 
+  ############################################################
+  # Functions for manipulating environment settings during tests:
   def backup_env
-    # Force a deep clone. Hacktacular, but works.
-    @_env = YAML.load(ENV.to_hash.to_yaml)
+    @_env = ENV.to_hash
   end
 
   def reduce_env(destroy_keys=[])
@@ -126,6 +125,10 @@ class SystemContext
 
   def restore_env
     if @_env
+      # delete environment variables we've added since we started
+      ENV.to_hash.each_pair {|k,v| ENV.delete(k) unless @_env.include?(k) }
+
+      # restore environment variables we've modified since we started
       @_env.each_pair {|k,v| ENV[k] = v}
     else
       raise InvalidBackupEnv.new
@@ -141,25 +144,74 @@ class SystemContext
       restore_env
     end
   end
+
+  ############################################################
+  # Functions for manipulating project.yml files during tests:
+  def merge_project_yml_for_test(settings, show_final=false)
+    yaml_wrapper = YamlWrapper.new
+    project_hash = yaml_wrapper.load('project.yml')
+    project_hash.deep_merge!(settings)
+    puts "\n\n#{project_hash.to_yaml}\n\n" if show_final
+    yaml_wrapper.dump('project.yml', project_hash)
+  end
+
+  def append_project_yml_for_test(new_args)
+    fake_prj_yml= "#{File.read('project.yml')}\n#{new_args}"
+    File.write('project.yml', fake_prj_yml, mode: 'w')
+  end
+
+  def uncomment_project_yml_option_for_test(option)
+    fake_prj_yml= File.read('project.yml').gsub(/\##{option}/,option)
+    File.write('project.yml', fake_prj_yml, mode: 'w')
+  end
+
+  def comment_project_yml_option_for_test(option)
+    fake_prj_yml= File.read('project.yml').gsub(/#{option}/,"##{option}")
+    File.write('project.yml', fake_prj_yml, mode: 'w')
+  end
+  ############################################################
 end
 
 module CeedlingTestCases
+  def can_report_version_no_git_commit_sha
+    @c.with_context do
+      # Version without Git commit short SHA file in project
+      output = `bundle exec ruby -S ceedling version 2>&1`
+      expect($?.exitstatus).to match(0)
+      expect(output).to match(/Ceedling => \d\.\d\.\d\n/)
+    end
+  end
+
+  def can_report_version_with_git_commit_sha
+    # Version with Git commit short SHA file in root of project
+    # Creating the commit file before building + installing the gem simulates the CI process
+    File.open('GIT_COMMIT_SHA', 'w') do |f|
+      f << '---{-@'
+    end
+
+    @c.with_context do
+      output = `bundle exec ruby -S ceedling version 2>&1`
+      expect($?.exitstatus).to match(0)
+      expect(output).to match(/Ceedling => \d\.\d\.\d----{-@\n/)
+    end
+  end
+
   def can_create_projects
     @c.with_context do
       Dir.chdir @proj_name do
-        expect(File.exists?("project.yml")).to eq true
-        expect(File.exists?("src")).to eq true
-        expect(File.exists?("test")).to eq true
-        expect(File.exists?("test/support")).to eq true
-        expect(File.exists?("test/support/.gitkeep")).to eq true
+        expect(File.exist?("project.yml")).to eq true
+        expect(File.exist?("src")).to eq true
+        expect(File.exist?("test")).to eq true
+        expect(File.exist?("test/support")).to eq true
       end
     end
   end
 
-  def has_an_ignore
+  def has_git_support
     @c.with_context do
       Dir.chdir @proj_name do
-        expect(File.exists?(".gitignore")).to eq true
+        expect(File.exist?(".gitignore")).to eq true
+        expect(File.exist?("test/support/.gitkeep")).to eq true
       end
     end
   end
@@ -168,20 +220,50 @@ module CeedlingTestCases
     @c.with_context do
       output = `bundle exec ruby -S ceedling upgrade #{@proj_name} 2>&1`
       expect($?.exitstatus).to match(0)
-      expect(output).to match(/upgraded!/i)
+      expect(output).to match(/Upgraded/i)
       Dir.chdir @proj_name do
-        expect(File.exists?("project.yml")).to eq true
-        expect(File.exists?("src")).to eq true
-        expect(File.exists?("test")).to eq true
+        expect(File.exist?("project.yml")).to eq true
+        expect(File.exist?("src")).to eq true
+        expect(File.exist?("test")).to eq true
         all_docs = Dir["vendor/ceedling/docs/*.pdf"].length + Dir["vendor/ceedling/docs/*.md"].length
       end
     end
   end
 
+  def can_upgrade_projects_even_if_test_support_folder_does_not_exist
+    @c.with_context do
+      output = `bundle exec ruby -S ceedling upgrade #{@proj_name} 2>&1`
+      FileUtils.rm_rf("#{@proj_name}/test/support")
+
+      updated_prj_yml = []
+      File.read("#{@proj_name}/project.yml").split("\n").each do |line|
+        updated_prj_yml.append(line) unless line =~ /support/
+      end
+      File.write("#{@proj_name}/project.yml", updated_prj_yml.join("\n"), mode: 'w')
+
+      expect($?.exitstatus).to match(0)
+      expect(output).to match(/Upgraded/i)
+      Dir.chdir @proj_name do
+        expect(File.exist?("project.yml")).to eq true
+        expect(File.exist?("src")).to eq true
+        expect(File.exist?("test")).to eq true
+        all_docs = Dir["vendor/ceedling/docs/*.pdf"].length + Dir["vendor/ceedling/docs/*.md"].length
+      end
+    end
+  end
+
+  def cannot_upgrade_non_existing_project
+    @c.with_context do
+      output = `bundle exec ruby -S ceedling upgrade #{@proj_name} 2>&1`
+      expect($?.exitstatus).to match(1)
+      expect(output).to match(/Could not find an existing project/i)
+    end
+  end
+
   def contains_a_vendor_directory
     @c.with_context do
       Dir.chdir @proj_name do
-        expect(File.exists?("vendor/ceedling")).to eq true
+        expect(File.exist?("vendor/ceedling")).to eq true
       end
     end
   end
@@ -189,7 +271,7 @@ module CeedlingTestCases
   def does_not_contain_a_vendor_directory
     @c.with_context do
       Dir.chdir @proj_name do
-        expect(File.exists?("vendor/ceedling")).to eq false
+        expect(File.exist?("vendor/ceedling")).to eq false
       end
     end
   end
@@ -206,7 +288,7 @@ module CeedlingTestCases
   def does_not_contain_documentation
     @c.with_context do
       Dir.chdir @proj_name do
-        expect(File.exists?("vendor/ceedling/docs")).to eq false
+        expect(File.exist?("vendor/ceedling/docs")).to eq false
         expect(Dir["vendor/ceedling/**/*.pdf"].length).to eq 0
       end
     end
@@ -263,6 +345,44 @@ module CeedlingTestCases
     end
   end
 
+  def can_test_projects_with_named_verbosity
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
+
+        output = `bundle exec ruby -S ceedling --verbosity=obnoxious 2>&1`
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+\d/)
+        expect(output).to match(/PASSED:\s+\d/)
+        expect(output).to match(/FAILED:\s+\d/)
+        expect(output).to match(/IGNORED:\s+\d/)
+
+        expect(output).to match (/:post_test_fixture_execute/)
+      end
+    end
+  end
+
+  def can_test_projects_with_numerical_verbosity
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
+
+        output = `bundle exec ruby -S ceedling -v=4 2>&1`
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+\d/)
+        expect(output).to match(/PASSED:\s+\d/)
+        expect(output).to match(/FAILED:\s+\d/)
+        expect(output).to match(/IGNORED:\s+\d/)
+
+        expect(output).to match (/:post_test_fixture_execute/)
+      end
+    end
+  end
+
   def can_test_projects_with_unity_exec_time
     @c.with_context do
       Dir.chdir @proj_name do
@@ -270,7 +390,7 @@ module CeedlingTestCases
         FileUtils.cp test_asset_path("example_file.c"), 'src/'
         FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
         settings = { :unity => { :defines => [ "UNITY_INCLUDE_EXEC_TIME" ] } }
-        add_project_settings("project.yml", settings)
+        @c.merge_project_yml_for_test(settings)
 
         output = `bundle exec ruby -S ceedling 2>&1`
         expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
@@ -289,9 +409,9 @@ module CeedlingTestCases
         FileUtils.cp test_asset_path("example_file.c"), 'src/'
         FileUtils.cp test_asset_path("test_example_file_unity_printf.c"), 'test/'
         settings = { :unity => { :defines => [ "UNITY_INCLUDE_PRINT_FORMATTED" ] },
-                     :defines => { :test_example_file_unity_printf => [ "TEST" ] }
+                     :defines => { :test => { :example_file_unity_printf => [ "TEST" ] } }
                    }
-        add_project_settings("project.yml", settings)
+        @c.merge_project_yml_for_test(settings)
 
         output = `bundle exec ruby -S ceedling 2>&1`
         expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
@@ -303,16 +423,19 @@ module CeedlingTestCases
     end
   end
 
-  def can_test_projects_with_enabled_auto_link_deep_deependency_with_success
+  def can_test_projects_with_test_name_replaced_defines_with_success
     @c.with_context do
       Dir.chdir @proj_name do
-        FileUtils.copy_entry test_asset_path("auto_link_deep_dependencies/src/"), 'src/'
-        FileUtils.cp_r test_asset_path("auto_link_deep_dependencies/test/."), 'test/'
-        settings = { :project => { :auto_link_deep_dependencies => true } }
-        add_project_settings("project.yml", settings)
+        FileUtils.copy_entry test_asset_path("tests_with_defines/src/"), 'src/'
+        FileUtils.cp_r test_asset_path("tests_with_defines/test/."), 'test/'
+        settings = { :defines => { :test => { '*' => [ "TEST", "STANDARD_CONFIG" ],
+                                   'test_adc_hardware_special.c' => [ "TEST", "SPECIFIC_CONFIG" ],
+                                 } }
+                   }
+        @c.merge_project_yml_for_test(settings)
 
         output = `bundle exec ruby -S ceedling 2>&1`
-        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect($?.exitstatus).to match(0) # Since a test either passes or is ignored, we return success here
         expect(output).to match(/TESTED:\s+\d/)
         expect(output).to match(/PASSED:\s+\d/)
         expect(output).to match(/FAILED:\s+\d/)
@@ -321,14 +444,15 @@ module CeedlingTestCases
     end
   end
 
-  def can_test_projects_with_enabled_preprocessor_directives_with_success
+  # Ceedling :use_test_preprocessor is disabled
+  def can_test_projects_unity_parameterized_test_cases_with_success
     @c.with_context do
       Dir.chdir @proj_name do
         FileUtils.cp test_asset_path("test_example_with_parameterized_tests.c"), 'test/'
-        settings = { :project => { :use_preprocessor_directives => true },
+        settings = { :project => { :use_test_preprocessor => :none },
                      :unity => { :use_param_tests => true }
                    }
-        add_project_settings("project.yml", settings)
+        @c.merge_project_yml_for_test(settings)
 
         output = `bundle exec ruby -S ceedling 2>&1`
         expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
@@ -340,16 +464,15 @@ module CeedlingTestCases
     end
   end
 
-  def can_test_projects_with_test_name_replaced_defines_with_success
+  # NOTE: This is not supported in this release, therefore is not getting called.
+  def can_test_projects_unity_parameterized_test_cases_with_preprocessor_with_success
     @c.with_context do
       Dir.chdir @proj_name do
-        FileUtils.copy_entry test_asset_path("tests_with_defines/src/"), 'src/'
-        FileUtils.cp_r test_asset_path("tests_with_defines/test/."), 'test/'
-        settings = { :defines => { :test => [ "STANDARD_CONFIG" ],
-                                   :test_adc_hardware_special => [ "TEST", "SPECIFIC_CONFIG" ]
-                                 }
+        FileUtils.cp test_asset_path("test_example_with_parameterized_tests.c"), 'test/'
+        settings = { :project => { :use_test_preprocessor => :all, :use_deep_preprocessor => :mocks },
+                     :unity => { :use_param_tests => true }
                    }
-        add_project_settings("project.yml", settings)
+        @c.merge_project_yml_for_test(settings)
 
         output = `bundle exec ruby -S ceedling 2>&1`
         expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
@@ -361,6 +484,113 @@ module CeedlingTestCases
     end
   end
 
+  def can_test_projects_with_preprocessing_for_test_files_symbols_undefined
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("tests_with_preprocessing/test/test_adc_hardwareA.c"), 'test/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareA.c"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareA.h"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardware_configuratorA.h"), 'src/'
+        # Rely on undefined symbols in our C files
+        # 2 enabled intentionally failing test cases (no mocks generated)
+        settings = { :project => { :use_test_preprocessor => :tests },
+                   }
+        @c.merge_project_yml_for_test(settings)
+
+        output = `bundle exec ruby -S ceedling test:adc_hardwareA 2>&1`
+        expect($?.exitstatus).to match(1) # Intentional test failure in successful build
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+0/)
+        expect(output).to match(/FAILED:\s+2/)
+      end
+    end
+  end
+
+  def can_test_projects_with_preprocessing_for_test_files_symbols_defined
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("tests_with_preprocessing/test/test_adc_hardwareA.c"), 'test/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareA.c"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareA.h"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardware_configuratorA.h"), 'src/'
+        # 1 enabled passing test case with 1 mock used
+        settings = { :project => { :use_test_preprocessor => :tests },
+                     :defines => { :test => ['PREPROCESSING_TESTS'] }
+                   }
+        @c.merge_project_yml_for_test(settings)
+
+        output = `bundle exec ruby -S ceedling test:adc_hardwareA 2>&1`
+        expect($?.exitstatus).to match(0) # Successful build and tests
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+      end
+    end
+  end
+
+  def can_test_projects_with_preprocessing_for_mocks_success
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("tests_with_preprocessing/test/test_adc_hardwareB.c"), 'test/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareB.c"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareB.h"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardware_configuratorB.h"), 'src/'
+        # 1 test case with 1 mocked function
+        settings = { :project => { :use_test_preprocessor => :mocks },
+                     :defines => { :test => ['PREPROCESSING_MOCKS'] }
+                   }
+        @c.merge_project_yml_for_test(settings)
+
+        output = `bundle exec ruby -S ceedling test:adc_hardwareB 2>&1`
+        expect($?.exitstatus).to match(0) # Successful build and tests
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+      end
+    end
+  end
+
+  def can_test_projects_with_preprocessing_for_mocks_intentional_build_failure
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("tests_with_preprocessing/test/test_adc_hardwareB.c"), 'test/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareB.c"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareB.h"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardware_configuratorB.h"), 'src/'
+        # 1 test case with a missing mocked function
+        settings = { :project => { :use_test_preprocessor => :mocks }
+                   }
+        @c.merge_project_yml_for_test(settings)
+
+        output = `bundle exec ruby -S ceedling test:adc_hardwareB 2>&1`
+        expect($?.exitstatus).to match(1) # Failing build because of missing mock
+        expect(output).to match(/(undeclared|undefined|implicit).+Adc_Reset/)
+      end
+    end
+  end
+
+  def can_test_projects_with_preprocessing_all
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("tests_with_preprocessing/test/test_adc_hardwareC.c"), 'test/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareC.c"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardwareC.h"), 'src/'
+        FileUtils.cp test_asset_path("tests_with_preprocessing/src/adc_hardware_configuratorC.h"), 'src/'
+        # 1 test case using 1 mock
+        settings = { :project => { :use_test_preprocessor => :all },
+                     :defines => { :test => ['PREPROCESSING_TESTS', 'PREPROCESSING_MOCKS'] }
+                   }
+        @c.merge_project_yml_for_test(settings)
+
+        output = `bundle exec ruby -S ceedling test:adc_hardwareC 2>&1`
+        expect($?.exitstatus).to match(0) # Successful build and tests
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+      end
+    end
+  end
+
   def can_test_projects_with_fail
     @c.with_context do
       Dir.chdir @proj_name do
@@ -421,25 +651,46 @@ module CeedlingTestCases
 
         output = `bundle exec ruby -S ceedling test:all 2>&1`
         expect($?.exitstatus).to match(1) # Since a test explodes, we return error here
-        expect(output).to match(/ERROR: Ceedling Failed/)
+        expect(output).to match(/(?:ERROR: Ceedling Failed)|(?:Ceedling could not complete operations because of errors)/)
+      end
+    end
+  end
+
+  def can_test_projects_with_both_mock_and_real_header
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("example_file_call.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file_call.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_with_mock.c"), 'test/'
+
+        output = `bundle exec ruby -S ceedling 2>&1`
+        expect($?.exitstatus).to match(0) # Since a test either passed or was ignored, we return success here
+        expect(output).to match(/TESTED:\s+\d/)
+        expect(output).to match(/PASSED:\s+\d/)
+        expect(output).to match(/FAILED:\s+\d/)
+        expect(output).to match(/IGNORED:\s+\d/)
       end
     end
   end
 
-  def uses_raw_output_report_plugin
+  def uses_report_tests_raw_output_log_plugin
     @c.with_context do
       Dir.chdir @proj_name do
         FileUtils.cp test_asset_path("example_file.h"), 'src/'
         FileUtils.cp test_asset_path("example_file.c"), 'src/'
         FileUtils.cp test_asset_path("test_example_file_verbose.c"), 'test/'
 
+        @c.uncomment_project_yml_option_for_test('- report_tests_raw_output_log')
+
         output = `bundle exec ruby -S ceedling test:all 2>&1`
         expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
         expect(output).to match(/TESTED:\s+\d/)
         expect(output).to match(/PASSED:\s+\d/)
         expect(output).to match(/FAILED:\s+\d/)
         expect(output).to match(/IGNORED:\s+\d/)
-        expect(File.exists?("build/artifacts/test/test_example_file_verbose.log")).to eq true
+        expect(File.exist?("build/artifacts/test/test_example_file_verbose.raw.log")).to eq true
       end
     end
   end
@@ -464,193 +715,280 @@ module CeedlingTestCases
         expect($?.exitstatus).to match(0)
         expect(output).to match(/ceedling clean/i)
         expect(output).to match(/ceedling clobber/i)
-        expect(output).to match(/ceedling logging/i)
         expect(output).to match(/ceedling module:create/i)
         expect(output).to match(/ceedling module:destroy/i)
         expect(output).to match(/ceedling summary/i)
         expect(output).to match(/ceedling test:\*/i)
         expect(output).to match(/ceedling test:all/i)
-        expect(output).to match(/ceedling test:delta/i)
         expect(output).to match(/ceedling version/i)
       end
     end
   end
 
-  def can_use_the_module_plugin
+  def can_run_single_test_with_full_test_case_name_from_test_file_with_success
     @c.with_context do
       Dir.chdir @proj_name do
-        output = `bundle exec ruby -S ceedling module:create[ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
-        output = `bundle exec ruby -S ceedling test:all`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Need to Implement ponies/)
-        output = `bundle exec ruby -S ceedling module:destroy[ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Destroy Complete/i)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        self.can_use_the_module_plugin_path_extension
-        self.can_use_the_module_plugin_with_include_path
+        output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=test_add_numbers_adds_numbers 2>&1`
+
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+0/)
       end
     end
   end
 
-  def can_use_the_module_plugin_path_extension
+  def can_run_single_test_with_partial_test_case_name_from_test_file_with_success
     @c.with_context do
       Dir.chdir @proj_name do
-        # Module creation
-        output = `bundle exec ruby -S ceedling module:create[myPonies:ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
-        expect(File.exists?("myPonies/src/ponies.c")).to eq true
-        expect(File.exists?("myPonies/src/ponies.h")).to eq true
-        expect(File.exists?("myPonies/test/test_ponies.c")).to eq true
-
-        # add module path to project file
-        settings = { :paths => { :test => [ "myPonies/test" ],
-                                 :source => [ "myPonies/src" ]
-                               }
-                   }
-        add_project_settings("project.yml", settings)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        # See if ceedling finds the test in the subdir
-        output = `bundle exec ruby -S ceedling test:all`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Need to Implement ponies/)
+        output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=_adds_numbers 2>&1`
 
-        # Module destruction
-        output = `bundle exec ruby -S ceedling module:destroy[myPonies:ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Destroy Complete/i)
-        expect(File.exists?("myPonies/src/ponies.c")).to eq false
-        expect(File.exists?("myPonies/src/ponies.h")).to eq false
-        expect(File.exists?("myPonies/test/test_ponies.c")).to eq false
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+0/)
       end
     end
   end
 
-  def can_use_the_module_plugin_with_include_path
+  def none_of_test_is_executed_if_test_case_name_passed_does_not_fit_defined_in_test_file
     @c.with_context do
       Dir.chdir @proj_name do
-        # add include path to module generator
-        mod_gen = Modulegenerator.new(inc_root: "inc/")
-        settings = { :module_generator => { :project_root => mod_gen.project_root,
-                                            :source_root => mod_gen.source_root,
-                                            :inc_root => mod_gen.inc_root,
-                                            :test_root => mod_gen.test_root
-                                          }
-                   }
-        add_project_settings("project.yml", settings)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        # module creation
-        output = `bundle exec ruby -S ceedling module:create[myPonies:ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
-        expect(File.exists?("myPonies/src/ponies.c")).to eq true
-        expect(File.exists?("myPonies/inc/ponies.h")).to eq true
-        expect(File.exists?("myPonies/test/test_ponies.c")).to eq true
+        output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=zumzum 2>&1`
 
-        # Module destruction
-        output = `bundle exec ruby -S ceedling module:destroy[myPonies:ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Destroy Complete/i)
-        expect(File.exists?("myPonies/src/ponies.c")).to eq false
-        expect(File.exists?("myPonies/inc/ponies.h")).to eq false
-        expect(File.exists?("myPonies/test/test_ponies.c")).to eq false
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/No tests executed./)
       end
     end
   end
 
-  def can_use_the_module_plugin_with_non_default_paths
+  def none_of_test_is_executed_if_test_case_name_and_exclude_test_case_name_is_the_same
     @c.with_context do
       Dir.chdir @proj_name do
-        # add paths to module generator
-        mod_gen = Modulegenerator.new({source_root: "foo/", inc_root: "bar/", test_root: "barz/"})
-        settings = { :module_generator => { :project_root => mod_gen.project_root,
-                                            :source_root => mod_gen.source_root,
-                                            :inc_root => mod_gen.inc_root,
-                                            :test_root => mod_gen.test_root
-                                          }
-                   }
-        add_project_settings("project.yml", settings)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        # module creation
-        output = `bundle exec ruby -S ceedling module:create[ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
-        expect(File.exists?("foo/ponies.c")).to eq true
-        expect(File.exists?("bar/ponies.h")).to eq true
-        expect(File.exists?("barz/test_ponies.c")).to eq true
+        output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=_adds_numbers --exclude_test_case=_adds_numbers 2>&1`
 
-        # Module destruction
-        output = `bundle exec ruby -S ceedling module:destroy[ponies]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Destroy Complete/i)
-        expect(File.exists?("foo/ponies.c")).to eq false
-        expect(File.exists?("bar/ponies.h")).to eq false
-        expect(File.exists?("barz/test_ponies.c")).to eq false
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/No tests executed./)
       end
     end
   end
 
-  def handles_creating_the_same_module_twice_using_the_module_plugin
+  def confirm_if_notification_for_cmdline_args_not_enabled_is_disabled
     @c.with_context do
       Dir.chdir @proj_name do
-        output = `bundle exec ruby -S ceedling module:create[unicorns]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        output = `bundle exec ruby -S ceedling module:create[unicorns] 2>&1`
-        expect($?.exitstatus).to match(1)
-        expect(output).to match(/ERROR: Ceedling Failed/)
+        output = `bundle exec ruby -S ceedling test:test_example_file_success 2>&1`
 
-        self.handles_creating_the_same_module_twice_using_the_module_plugin_path_extension
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+1/)
+        expect(output).not_to match(/:cmdline_args/)
       end
     end
   end
 
-  def handles_creating_the_same_module_twice_using_the_module_plugin
+  def exclude_test_case_name_filter_works_and_only_one_test_case_is_executed
     @c.with_context do
       Dir.chdir @proj_name do
-        output = `bundle exec ruby -S ceedling module:create[myUnicorn:unicorns]`
-        expect($?.exitstatus).to match(0)
-        expect(output).to match(/Generate Complete/i)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
+
+        output = `bundle exec ruby -S ceedling test:all --exclude_test_case=test_add_numbers_adds_numbers 2>&1`
 
-        output = `bundle exec ruby -S ceedling module:create[myUnicorn:unicorns] 2>&1`
-        expect($?.exitstatus).to match(1)
-        expect(output).to match(/ERROR: Ceedling Failed/)
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+0/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+1/)
       end
     end
   end
 
-  def handles_destroying_a_module_that_does_not_exist_using_the_module_plugin
+  def run_one_testcase_from_one_test_file_when_test_case_name_is_passed
     @c.with_context do
       Dir.chdir @proj_name do
-        output = `bundle exec ruby -S ceedling module:destroy[unknown]`
-        expect($?.exitstatus).to match(0)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/'
 
-        expect(output).to match(/File src\/unknown\.c does not exist so cannot be removed\./)
-        expect(output).to match(/File src\/unknown\.h does not exist so cannot be removed\./)
-        expect(output).to match(/File test\/test_unknown\.c does not exist so cannot be removed\./)
-        expect(output).to match(/Destroy Complete/)
+        output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=_adds_numbers 2>&1`
 
-        self.handles_destroying_a_module_that_does_not_exist_using_the_module_plugin_path_extension
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+0/)
       end
     end
   end
 
-  def handles_destroying_a_module_that_does_not_exist_using_the_module_plugin_path_extension
+
+  def test_run_of_projects_fail_because_of_crash_without_report
     @c.with_context do
       Dir.chdir @proj_name do
-        output = `bundle exec ruby -S ceedling module:destroy[myUnknownModule:unknown]`
-        expect($?.exitstatus).to match(0)
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/'
+
+        @c.merge_project_yml_for_test({:project => { :use_backtrace => :none }})
+
+        output = `bundle exec ruby -S ceedling test:all 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail because of crash
+        expect(output).to match(/Test Executable Crashed/i)
+        expect(output).to match(/Unit test failures./)
+        expect(!File.exist?('./build/test/results/test_add.fail'))
+      end
+    end
+  end
+
+  def test_run_of_projects_fail_because_of_crash_with_report
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/'
+
+        @c.merge_project_yml_for_test({:project => { :use_backtrace => :none }})
+
+        output = `bundle exec ruby -S ceedling test:all 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail because of crash
+        expect(output).to match(/Test Executable Crashed/i)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_crash.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_crash.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ )
+      end
+    end
+  end
 
-        expect(output).to match(/File myUnknownModule\/src\/unknown\.c does not exist so cannot be removed\./)
-        expect(output).to match(/File myUnknownModule\/src\/unknown\.h does not exist so cannot be removed\./)
-        expect(output).to match(/File myUnknownModule\/test\/test_unknown\.c does not exist so cannot be removed\./)
-        expect(output).to match(/Destroy Complete/)
+  def execute_all_test_cases_from_crashing_test_runner_and_return_test_report_with_failue
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/'
+
+        @c.merge_project_yml_for_test({:project => { :use_backtrace => :gdb }})
+
+        output = `bundle exec ruby -S ceedling test:all 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail because of crash
+        expect(output).to match(/Test Case Crashed/i)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_crash.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_crash.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ )
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+(?:0|1)/)
+        expect(output).to match(/FAILED:\s+(?:1|2)/)
+        expect(output).to match(/IGNORED:\s+0/)
       end
     end
   end
 
+  def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_test_case_argument_with_enabled_debug
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/'
+
+        @c.merge_project_yml_for_test({:project => { :use_backtrace => :gdb }})
+
+        output = `bundle exec ruby -S ceedling test:all --test_case=test_add_numbers_will_fail 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail because of crash
+        expect(output).to match(/Test Case Crashed/i)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_crash.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_crash.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ )
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+(?:0|1)/)
+        expect(output).to match(/FAILED:\s+(?:1|2)/)
+        expect(output).to match(/IGNORED:\s+0/)
+      end
+    end
+  end
+
+  def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_exclude_test_case_argument_with_enabled_debug
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/'
+
+        @c.merge_project_yml_for_test({:project => { :use_backtrace => :gdb }})
+
+        output = `bundle exec ruby -S ceedling test:all --exclude_test_case=add_numbers_adds_numbers 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail because of crash
+        expect(output).to match(/Test Case Crashed/i)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_crash.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_crash.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ )
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+(?:0|1)/)
+        expect(output).to match(/FAILED:\s+(?:1|2)/)
+        expect(output).to match(/IGNORED:\s+0/)
+      end
+    end
+  end
+
+  def can_test_projects_with_success_when_space_appears_between_hash_and_include
+    # test case cover issue described in https://github.com/ThrowTheSwitch/Ceedling/issues/588
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path('test_example_file_success.c'), 'test/'
+
+        add_line = false
+        updated_test_file = []
+        File.read(File.join('test','test_example_file_success.c')).split("\n").each do |line|
+          if line =~ /#include "unity.h"/
+            add_line = true
+            updated_test_file.append(line)
+          else
+            if add_line
+              updated_test_file.append('# include "unity.h"')
+              add_line = false
+            end
+            updated_test_file.append(line)
+          end
+        end
+
+        File.write(File.join('test','test_example_file_success.c'), updated_test_file.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling 2>&1`
+        expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here
+        expect(output).to match(/TESTED:\s+\d/)
+        expect(output).to match(/PASSED:\s+\d/)
+        expect(output).to match(/FAILED:\s+\d/)
+        expect(output).to match(/IGNORED:\s+\d/)
+      end
+    end
+  end
 end