ceedling 0.31.1 → 1.0.1

data/lib/ceedling/preprocessinator_includes_handler.rb

@@ -1,189 +1,382 @@
-
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
 
 class PreprocessinatorIncludesHandler
 
-  constructor :configurator, :tool_executor, :task_invoker, :file_path_utils, :yaml_wrapper, :file_wrapper, :file_finder
-  @@makefile_cache = {}
+  constructor :configurator, :tool_executor, :test_context_extractor, :file_wrapper, :yaml_wrapper, :loginator, :reportinator
+
+  ##
+  ## Includes Extraction Overview
+  ## ============================
+  ##
+  ## BACKGROUND
+  ## --------
+  ## #include extraction is hard to do. In simple cases a regex approach suffices, but nested header files,
+  ## clever macros, and conditional preprocessing statements easily introduce high complexity.
+  ##
+  ## Unfortunately, there's no readily available cross-platform C parsing tool that provides a simple means 
+  ## to extract the #include statements directly embedded in a given file. Even the gcc preprocessor itself 
+  ## only comes close to providing this information externally.
+  ##
+  ## APPROACH
+  ## --------
+  ## (Full details including fallback options are in the extensive code comments among the methods below.)
+  ## 
+  ## Sadly, we can't preprocess a file with full search paths and defines and ask for the #include statements
+  ## embedded in a file. We get far more #includes than we want with no way to discern which are at the depth
+  ## of the file being processed.
+  ##
+  ## Instead, we try our best to use some educated guessing to get as close as possible to the desired list.
+  ##
+  ##   I. Try to extract shallow defines with no crawling out into other header files. This conservative approach
+  ##      gives us a reference point on possible directly included files. The results may be incomplete, though. 
+  ##      They also may mistakenly list #includes that should not be in the list--because of #ifndef defaults or
+  ##      because of system headers or #include <...> statements and differences among gcc implementations.
+  ##
+  ##  II. Extract a full list of #includes by spidering out into nested headers and processing all macros, etc.
+  ##      This is the greedy approach.
+  ##
+  ## III. Find #includes common to (I) and (II). The results of (I) should limit the potentially lengthy
+  ##      results of (II). The complete and accurate list of (II) should cut out any mistaken entries in (I).
+  ##
+  ##  IV. I–III are not foolproof. A purely greedy approach or a purely conservative approach will cause symbol 
+  ##      conflicts, missing symbols, etc. The blended and balanced approach should come quite close to an 
+  ##      accurate list of shallow includes. Edge cases and gaps will cause trouble. Other Ceedling features 
+  ##      should provide the tools to intervene. 
+  ##
+
+  def extract_includes(filepath:, test:, flags:, include_paths:, defines:, deep: false)
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting #include statements via preprocessor from",
+      module_name: test,
+      filename: File.basename(filepath)
+      )
+    @loginator.log(msg, Verbosity::NORMAL)
+
+    # Extract shallow includes with preprocessor and fallback regex
+    shallow = extract_shallow_includes(
+      test:     test,
+      filepath: filepath,
+      flags:    flags,
+      defines:  defines
+      )
+
+    # Extract nested includes but optionally act in fallback mode
+    nested = extract_nested_includes(
+      filepath:      filepath,
+      include_paths: include_paths,
+      flags:         flags,
+      defines:       defines,
+      # If no shallow results, fall back to only depth 1 results of nested discovery
+      shallow:       shallow.empty?
+      )
 
-  # shallow includes: only those headers a source file explicitly includes
+    # Combine shallow and nested include knowledge of mocks
+    mocks = combine_mocks(shallow, nested)
 
-  def invoke_shallow_includes_list(filepath)
-    @task_invoker.invoke_test_shallow_include_lists( [@file_path_utils.form_preprocessed_includes_list_filepath(filepath)] )
+    # Redefine shallow and nested results without any mocks
+    shallow = remove_mocks( shallow )
+    nested  = remove_mocks( nested )
+
+    # Return
+    #  - Includes common to shallow and nested results, with paths from nested
+    #  - Add mocks back in (may be empty if mocking not enabled)
+    return common_includes(shallow:shallow, nested:nested, deep:deep) + mocks
   end
 
-  ##
-  # Ask the preprocessor for a make-style dependency rule of only the headers
-  # the source file immediately includes.
-  #
-  # === Arguments
-  # +filepath+ _String_:: Path to the test file to process.
-  #
-  # === Return
-  # _String_:: The text of the dependency rule generated by the preprocessor.
-  def form_shallow_dependencies_rule(filepath)
-    if @@makefile_cache.has_key?(filepath)
-      return @@makefile_cache[filepath]
+  # Write to disk a yaml representation of a list of includes
+  def write_includes_list(filepath, list)
+    @yaml_wrapper.dump(filepath, list)
+  end
+
+  ### Private ###
+  private
+
+  def extract_shallow_includes(test:, filepath:, flags:, defines:)
+    # Shallow includes extraction, first attempt with preprocessor
+    success, shallow = 
+      extract_shallow_includes_preprocessor(
+        test:     test,
+        filepath: filepath,
+        flags:    flags,
+        defines:  defines
+        )
+
+    # Shallow includes extraction, second attempt with file read + regex
+    if not success
+      shallow = extract_shallow_includes_regex(
+        test:     test,
+        filepath: filepath,
+        flags:    flags,
+        defines:  defines
+        )
     end
-    # change filename (prefix of '_') to prevent preprocessor from finding
-    # include files in temp directory containing file it's scanning
-    temp_filepath = @file_path_utils.form_temp_path(filepath, '_')
-
-    # read the file and replace all include statements with a decorated version
-    # (decorating the names creates file names that don't exist, thus preventing
-    # the preprocessor from snaking out and discovering the entire include path
-    # that winds through the code). The decorated filenames indicate files that
-    # are included directly by the test file.
-    contents = @file_wrapper.read(filepath)
-
-    if !contents.valid_encoding?
-      contents = contents.encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
+
+    return shallow
+  end
+
+  def extract_shallow_includes_preprocessor(test:, filepath:, flags:, defines:)
+    ##
+    ## Preprocessor Make Rule Handling
+    ## ===============================
+    ##
+    ## Creation:
+    ##  - This output is created with the -MM -MG -MP command line options.
+    ##  - No search paths are used towards extracting only the #include statements of the file.
+    ##    The intent is to minimize the list of .h -> .c module matches to, in turn, minimize
+    ##    unnecessary compilation when extracting includes from a test file.
+    ##  - Note: This approach can have gaps with complex macros / conditional statements.
+    ##          Gaps can be minimized with proper defines in the project file.
+    ##          However, needed / complex macros located in other header files can still gum 
+    ##          up the works.
+    ##
+    ## Format:
+    ##  - First line is .o file followed by colon and dependencies (on one or more lines).
+    ##  - "Phony" make rules follow that conveniently list each #include, one per line.
+    ##
+    ## Notes:
+    ##  - Many errors can occur but may not necessarily prevent usable results.
+    ##  - A file with no includes will create the first line with self-referential .h file path.
+    ##  - Make rule formation assumes any files not found in a search path will be generated.
+    ##    - Since we're not using search paths, the preprocessor largely assumes all #include 
+    ##      files are generated (and include no paths).
+    ##    - The exception is #include files that exist in the same directory as the file
+    ##      being processed.
+    ##
+    ## Approach:
+    ##  1. Disable exceptions for tool execution as errors are likely.
+    ##    - We may still have usable output.
+    ##    - We do not want to stop execution on fatal error; instead use a fallback method.
+    ##  2. The only true error is no make rule present--check for this first.
+    ##    - A make rule may be present but not depedencies if the file has no #includes.
+    ##  3. Extract includes from "phony" make rules that follow opening rule line.
+    ##    - These may be .h or .c files.
+    ## 
+    ## Example output follows
+    ## -----------------------------------------------------------------------------------------
+    ## os.o: ../../src/app/task/os/os.h fstd_types.h FreeRTOS.h queue.h
+    ## fstd_types.h:
+    ## FreeRTOS.h:
+    ## queue.h:
+    ## ../../src/app/task/os/os.h:72:21: error: no include path in which to search for stdbool.h
+    ##    72 | #include <stdbool.h>
+    ##       |                     ^
+    ## ../../src/app/task/os/os.h:73:20: error: no include path in which to search for stdint.h
+    ##    73 | #include <stdint.h>
+    ##       |                    ^
+    ##
+
+    # Matcher for the first line of the make rule output
+    make_rule_matcher = /^\S+\.o:\s+.+$/  # <characters>.o: <characters>
+    
+    # Matcher for the “phony“ make rule output lines for each #include dependency (.h, .c, etc.)
+    # Capture file name before the colon
+    include_matcher   = /^(\S+\.\S+):\s*$/ # <characters>.<extension>:
+
+    command = 
+      @tool_executor.build_command_line(
+        @configurator.tools_test_shallow_includes_preprocessor,
+        flags,
+        filepath,
+        defines
+        )
+
+    # Assume possible errors so we have best shot at extracting results from preprocessing.
+    # Full code compilation will catch any breaking code errors
+    command[:options][:boom] = false
+    shell_result = @tool_executor.exec( command )
+
+    make_rules = shell_result[:output]
+
+    # Do not check exit code for success. In some error conditions we still get usable output.
+    # Look for the first line of the make rule output.
+    if not make_rules =~ make_rule_matcher
+      msg = "Preprocessor #include extraction failed: #{shell_result[:output]}"
+      @loginator.log(msg, Verbosity::DEBUG)
+
+      return false, []
     end
 
-    contents.gsub!( /^\s*#include\s+[\"<]\s*(\S+)\s*[\">]/, "#include \"\\1\"\n#include \"@@@@\\1\"" )
-    contents.gsub!( /^\s*TEST_FILE\(\s*\"\s*(\S+)\s*\"\s*\)/, "#include \"\\1\"\n#include \"@@@@\\1\"")
-    @file_wrapper.write( temp_filepath, contents )
+    includes = []
 
-    # extract the make-style dependency rule telling the preprocessor to
-    # ignore the fact that it can't find the included files
-    command = @tool_executor.build_command_line(@configurator.tools_test_includes_preprocessor, [], temp_filepath)
-    shell_result = @tool_executor.exec(command[:line], command[:options])
+    # Extract the #include dependencies from the "phony" make rules, one per line
+    includes = make_rules.scan( include_matcher )
+    includes.flatten! # Regex results can be nested arrays becuase of paren captures
 
-    @@makefile_cache[filepath] = shell_result[:output]
-    return shell_result[:output]
+    return true, includes.uniq
   end
 
-  ##
-  # Extract the headers that are directly included by a source file using the
-  # provided, annotated Make dependency rule.
-  #
-  # === Arguments
-  # +filepath+ _String_:: C source or header file to extract includes for.
-  #
-  # === Return
-  # _Array_ of _String_:: Array of the direct dependencies for the source file.
-  def extract_includes(filepath)
-    to_process = [filepath]
-    ignore_list = []
-    list = []
-    all_mocks = []
-
-    include_paths = @configurator.project_config_hash[:collection_paths_include]
-    include_paths = [] if include_paths.nil?
-    include_paths.map! {|path| File.expand_path(path)}
-
-    while to_process.length > 0
-      target = to_process.shift()
-      ignore_list << target
-      new_deps, new_to_process, all_mocks = extract_includes_helper(target, include_paths, ignore_list, all_mocks)
-      list += new_deps
-      to_process += new_to_process
-      if !@configurator.project_config_hash[:project_auto_link_deep_dependencies]
-        break
-      else
-        list = list.uniq()
-        to_process = to_process.uniq()
-      end
+  def extract_shallow_includes_regex(test:, filepath:, flags:, defines:)
+    msg = @reportinator.generate_module_progress(
+      operation: "Using fallback regex #include extraction for",
+      module_name: test,
+      filename: File.basename( filepath )
+      )
+    @loginator.log(msg, Verbosity::NORMAL)
+
+    # Use abilities of @test_context_extractor to extract the #includes via regex on the file
+    includes = []
+    @file_wrapper.open( filepath, 'r' ) do |file|
+      includes = @test_context_extractor.extract_includes( file )
     end
 
-    return list
+    return includes
   end
 
-  def extract_includes_helper(filepath, include_paths, ignore_list, mocks)
-    # Extract the dependencies from the make rule
-    make_rule = self.form_shallow_dependencies_rule(filepath)
-    target_file = make_rule.split[0].gsub(':', '').gsub('\\','/')
-    base = File.basename(target_file, File.extname(target_file))
-    make_rule_dependencies = make_rule.gsub(/.*\b#{Regexp.escape(base)}\S*/, '').gsub(/\\$/, '')
-    
-    # Extract the headers dependencies from the make rule
-    hdr_ext = @configurator.extension_header
-    headers_dependencies = make_rule_dependencies.split.find_all {|path| path.end_with?(hdr_ext) }.uniq
-    headers_dependencies.map! {|hdr| hdr.gsub('\\','/') }
-    full_path_headers_dependencies = extract_full_path_dependencies(headers_dependencies)
-
-    # Extract the sources dependencies from the make rule
-    src_ext = @configurator.extension_source
-    sources_dependencies = make_rule_dependencies.split.find_all {|path| path.end_with?(src_ext) }.uniq
-    sources_dependencies.map! {|src| src.gsub('\\','/') }
-    full_path_sources_dependencies = extract_full_path_dependencies(sources_dependencies)
-
-    list = full_path_headers_dependencies + full_path_sources_dependencies
-
-    mock_prefix = @configurator.project_config_hash[:cmock_mock_prefix]
-    # Creating list of mocks
-    mocks += full_path_headers_dependencies.find_all do |header|
-      File.basename(header) =~ /^#{mock_prefix}.*$/
-    end.compact
-
-    # ignore real file when both mock and real file exist
-    mocks.each do |mock|
-      list.each do |filename|
-        if File.basename(filename) == File.basename(mock).sub(mock_prefix, '')
-          ignore_list << filename
-        end
-      end
-    end.compact
-
-    # Filtering list of final includes to only include mocks and anything that is NOT in the ignore_list
-    list = list.select do |item|
-      mocks.include? item or !(ignore_list.any? { |ignore_item| !item.match(/^(.*\/)?#{Regexp.escape(ignore_item)}$/).nil? })
-    end
+  def extract_nested_includes(filepath:, include_paths:, flags:, defines:, shallow:false)
+    ##
+    ## Preprocessor Header File Listing Handling
+    ## =========================================
+    ##
+    ## Creation:
+    ##  - This output is created with the -MM -MG -H command line options.
+    ##    - -MM -MG generates unused make rule that significantly reduces overall output.
+    ##    - -H creates the header file output listing we actually want.
+    ##  - Search paths are provided towards fully preprocessing all macros / conditionals and
+    ##    symbols. (This produces a rich list of #includes far greater than we need.)
+    ##
+    ## Format (ignoring throwaway make rule):
+    ##  - Each included filepath is listed per line.
+    ##  - The depth of the #include nesting is signified by precending '.'s.
+    ##  - Files directly #include'd in the file being preprocessed are at depth 1 ('.')
+    ##
+    ## Notes:
+    ##  - Because search paths and defines are provided, error-free execution is assumed.
+    ##    If the preprocessor fails, issues exist that will cause full compilation to fail.
+    ##  - Unfortuantely, because of ordering and nesting effects, a file directly #include'd may
+    ##    not be listed at depth 1 ('.'). Instead, it may end up listed at greater depth beneath 
+    ##    another #include'd file if both files reference it. That is, there is no way
+    ##    to give the preprocessor full context and ask for only the files directly 
+    ##    #include'd in the file being processed.
+    ##  - The preprocessor outputs the -H #include listing to STDERR. ToolExecutor does this
+    ##    by default in creating the shell result output.
+    ##  - Since we're using search paths, all #included files will include paths. Depending on
+    ##    circumstances, this could yield a list with generated mocks with full build paths.
+    ##
+    ## Approach:
+    ##  - Match on each listing line a filepath preceeded by its depth
+    ##  - One mode of using this preprocessor approach is as a fallback / double-check method 
+    ##    if the simpler, earler shallow preprocessing produces no #include results. When used
+    ##    this way we match only #include'd files at depth 1 ('.'), hoping we extract an
+    ##    appropriate, usable list of #includes.
+    ## 
+    ## Example output follows
+    ## -----------------------------------------------------------------------------------------
+    ## . build/vendor/unity/src/unity.h
+    ## .. build/vendor/unity/src/unity_internals.h
+    ## . src/Types.h
+    ## . src/Model.h
+    ## . src/TimerModel.h
+    ## .. src/Testing.h
+    ## TestModel.o: test/TestModel.c build/vendor/unity/src/unity.h \
+    ##   build/vendor/unity/src/unity_internals.h setjmp.h math.h stddef.h \
+    ##   stdint.h limits.h stdio.h src/Types.h src/Model.h src/TimerModel.h \
+    ##   src/Testing.h MockTaskScheduler.h MockTemperatureFilter.h
+    ##
+
+    command = 
+      @tool_executor.build_command_line(
+        @configurator.tools_test_nested_includes_preprocessor,
+        flags,
+        filepath,
+        include_paths,
+        defines
+      )
 
-    to_process = []
-
-    if @configurator.project_config_hash[:project_auto_link_deep_dependencies]
-      # Creating list of headers that should be recursively pre-processed
-      # Skipping mocks and vendor headers
-      headers_to_deep_link = full_path_headers_dependencies.select do |hdr|
-        !(mocks.include? hdr) and (hdr.match(/^(.*\/)(#{VENDORS_FILES.join('|')}) + #{Regexp.escape(hdr_ext)}$/).nil?)
-      end
-      headers_to_deep_link.map! {|hdr| File.expand_path(hdr) }
-      headers_to_deep_link.compact!
-
-      headers_to_deep_link.each do |hdr|
-        if (ignore_list.none? {|ignore_header| hdr.match(/^(.*\/)?#{Regexp.escape(ignore_header)}$/)} and
-            include_paths.none? {|include_path| hdr =~ /^#{include_path}\.*/})
-          if File.exist?(hdr)
-            to_process << hdr
-            src = @file_finder.find_compilation_input_file(hdr, :ignore)
-            to_process << src if src
-          end
-        end
-      end
+    # Let the preprocessor do as much as possible
+    # We'll extract nothing if a catastrophic error, but we'll see it in debug logging
+    # Any real problems will be flagged by actual compilation step
+    command[:options][:boom] = false
+
+    shell_result = @tool_executor.exec( command )
+
+    list = shell_result[:output]
+
+    includes = []
+
+    # Extract entries from #include listing
+    if shallow
+      # First level of includes in preprocessor output
+      includes = list.scan(/^\. (.+$)\s*$/)  # . <filepath>
+    else
+      # All levels of includes in preprocessor output
+      includes = list.scan(/^\.+ (.+$)\s*$/) # ... <filepath>
     end
 
-    return list, to_process, mocks
+    includes.flatten! # Regex results can be nested arrays becuase of paren captures
 
+    return includes.uniq
   end
 
-  def write_shallow_includes_list(filepath, list)
-    @yaml_wrapper.dump(filepath, list)
+  def combine_mocks(*lists)
+    # Handle mocks
+    #  - Ensure no build filepaths in mock listings
+    #  - Do not return mocks if mocking is disabled
+    mocks = []
+
+    # Bail out early if mocks are not enabled
+    return [] if !@configurator.project_use_mocks
+
+    # Use some greediness to ensure we get all possible mocks
+    lists.each { |list| mocks |= extract_mocks( list ) }      
+
+    # If generated mocks are in the build directory, the preprocessor will have found them.
+    # This leads to duplicated mocks -- the shallow list with no paths and the nested list with paths.
+    # Remove mocks with any path (the path will be the build directory); preserve just the shallow list.
+    mocks.reject! {|mock| File.dirname( mock ) != '.' }
+
+    return mocks
   end
 
-  private
+  # Return a list of mock .h files with no paths
+  def extract_mocks(includes)
+    return includes.select { |include| File.basename(include).start_with?( @configurator.cmock_mock_prefix ) }
+  end
+
+  # Return list of includes with any mocks removed
+  def remove_mocks(includes)
+    return includes.reject { |include| File.basename(include).start_with?( @configurator.cmock_mock_prefix ) }
+  end
+
+  # Return includes common in both lists with the full paths of the nested list
+  def common_includes(shallow:, nested:, deep: false)
+    return shallow if nested.empty?
+    return nested if shallow.empty?
 
-  def extract_full_path_dependencies(dependencies)
-    # Separate the real files form the annotated ones and remove the '@@@@'
-    annotated_files, real_files = dependencies.partition {|file| file =~ /^@@@@/}
-    annotated_files.map! {|file| file.gsub('@@@@','') }
-    # Matching annotated_files values against real_files to ensure that
-    # annotated_files contain full path entries (as returned by make rule)
-    annotated_files.map! {|file| real_files.find {|real| !real.match(/^(.*\/)?#{Regexp.escape(file)}$/).nil?}}
-    annotated_files = annotated_files.compact
-
-    # Find which of our annotated files are "real" dependencies. This is
-    # intended to weed out dependencies that have been removed due to build
-    # options defined in the project yaml and/or in the files themselves.
-    return annotated_files.find_all do |annotated_file|
-      # find the index of the "real" file that matches the annotated one.
-      idx = real_files.find_index do |real_file|
-        real_file =~ /^(.*\/)?#{Regexp.escape(annotated_file)}$/
-      end
-      # If we found a real file, delete it from the array and return it,
-      # otherwise return nil. Since nil is falsy this has the effect of making
-      # find_all return only the annotated filess for which a real file was
-      # found/deleted
-      idx ? real_files.delete_at(idx) : nil
-    end.compact
+    # Notes:
+    #  - We want to preserve filepaths whenever possible. Other areas of Ceedling use or discard the 
+    #    filepath as needed.
+    #  - We generally do not have filepaths in the shallow list--except when the #include is in the 
+    #    same directory as the file being processed
+
+    # Approach
+    #  1. Create hashed lists of shallow and nested for easier matching 
+    #  2. Perform appropriate mix of paths
+    #    a. A union if performing a deep include list
+    #    b. An intersection if performing a shallow include list
+    #  3. Pick the "fullest" path from the lists (assumes nested list has deeper paths)
+
+    # Hash list for Shallow Search
+    _shallow = {}
+    shallow.each { |item| _shallow[ File.basename(item) ] = item }
+
+    # Hash list for Nested Search
+    _nested = {}
+    nested.each {|item|  _nested[ File.basename(item) ] = item }
+
+    # Determine the filenames to include in our list
+    basenames = if deep
+      ( _nested.keys.to_set.union( _shallow.keys.to_set ) )
+    else
+      ( _nested.keys.to_set.intersection( _shallow.keys.to_set ) )
+    end
+
+    # Iterate through the basenames and return the fullest version of each
+    return basenames.map {|v| _nested[v] || _shallow[v] }
   end
+
 end

data/lib/ceedling/rake_utils.rb

@@ -1,3 +1,9 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
 
 class RakeUtils
   

data/lib/ceedling/rake_wrapper.rb

@@ -1,3 +1,10 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
 require 'rubygems'
 require 'rake'
 require 'ceedling/makefile' # our replacement for rake's make-style dependency loader