ceedling 1.0.1 → 1.1.0

data/lib/ceedling/preprocess/preprocessinator_includes_handler.rb

@@ -0,0 +1,224 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/includes/includes'
+require 'ceedling/preprocess/preprocessinator_bare_includes_extractor'
+require 'ceedling/preprocess/preprocessinator_line_marker_includes_extractor'
+require 'ceedling/preprocess/c_preprocessor_conditionals'
+require 'ceedling/exceptions'
+
+class PreprocessinatorIncludesHandler
+
+  constructor(
+    :configurator,
+    :preprocessinator_line_marker_includes_extractor,
+    :include_factory,
+    :tool_executor,
+    :file_wrapper,
+    :yaml_wrapper,
+    :parsing_parcels,
+    :loginator,
+    :reportinator
+  )
+
+  def setup()
+    # Aliases
+    @line_marker_includes_extractor = @preprocessinator_line_marker_includes_extractor
+  end
+
+  def extract_bare_includes(test:, filepath:, search_paths:, flags:, defines:)
+    filename = File.basename(filepath)
+
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting bare #includes via preprocessing from",
+      module_name: test,
+      filename: filename
+    )
+    @loginator.log( msg, Verbosity::OBNOXIOUS )
+
+    # Creation:
+    #  - This output is created with the -MM -MG -MP command line options.
+    #  - Limited search paths are used towards shallow extracting of only the user #include statements of the file.
+    #    This preprocessor mode assumes any includes discovered outside of a search path will be generated.
+    #
+    # Notes:
+    #  - This approach can have gaps with advacnced user-level macros like `#include <MACRO>`.
+    #    By including Ceedling's vendor search path, we support Partials macros of this sort.
+    #  - Gaps can be minimized with proper defines in the project file. However, needed, complex macros 
+    #    located in other header files could still gum up the works.
+    #  - Many errors can occur but may not necessarily prevent usable results.
+    command = 
+      @tool_executor.build_command_line(
+        @configurator.tools_test_bare_includes_preprocessor,
+        # No additional arguments
+        [],
+        # Argument replacement
+        filepath,
+        defines,
+        flags,
+        search_paths
+      )
+
+    # Assume possible errors so we have best shot at extracting results from preprocessing.
+    # Full code compilation will catch any breaking code errors
+    command[:options][:boom] = false
+    shell_result = @tool_executor.exec( command )
+
+    make_rules = shell_result[:output]
+
+    # Do not check exit code for success. In some error conditions we still get usable output.
+    # Look for the first line of the make rule output.
+    if not make_rules =~ PreprocessinatorBareIncludesExtractor::MAKE_RULE_MATCHER
+      @loginator.lazy( Verbosity::DEBUG ) do
+        "Preprocessor bare #include extraction failed: #{shell_result[:output]}"
+      end
+      return []
+    end
+
+    includes = PreprocessinatorBareIncludesExtractor.extract_includes( make_rules )
+    includes = clean_self_reference(filepath, includes)
+
+    header = "Extracted bare #includes from #{filepath}:"
+    @loginator.log_list( includes, header, Verbosity::DEBUG )
+
+    return includes
+  end
+
+  def extract_user_includes_preprocess(name:, filepath:, preprocessed_filepath:)
+    includes = []
+
+    filename = File.basename(filepath)
+
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting user #includes from preprocessed output",
+      module_name: name,
+      filename: filename
+    )
+    @loginator.log(msg, Verbosity::OBNOXIOUS)
+
+    includes = 
+      @line_marker_includes_extractor.extract_includes_from_file(
+        preprocessed_filepath,
+        PreprocessinatorLineMarkerIncludesExtractor::USER
+        # Note: No limit to max depth to search for user includes
+      )
+
+    return clean_self_reference( filepath, includes )
+  end
+
+  def extract_user_includes_from_text(name:, filepath:, defines: [])
+    includes = []
+
+    filename = File.basename(filepath)
+
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting user #includes from original file using fallback method",
+      module_name: name,
+      filename: filename
+    )
+    @loginator.log( msg, Verbosity::OBNOXIOUS, LogLabels::WARNING )
+
+    cond_tracker = CPreprocessorConditionals.new( defines )
+
+    # Open in binary mode: code_lines applies clean_encoding per-line, but each_line
+    # itself can raise on invalid byte sequences before clean_encoding is reached.
+    @file_wrapper.open(filepath, 'rb') do |input|
+      @parsing_parcels.code_lines( input ) do |line|
+        cond_tracker.process_directive( line )
+        next unless cond_tracker.active?
+        _include = @include_factory.user_include_from_directive( line )
+        includes << _include if !_include.nil?
+      end
+    end
+
+    return clean_self_reference( filepath, includes )
+  end
+
+  def extract_system_includes_preprocess(name:, filepath:, preprocessed_filepath:)
+    includes = []
+
+    filename = File.basename(filepath)
+
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting system #includes from preprocessed output",
+      module_name: name,
+      filename: filename
+    )
+    @loginator.log(msg, Verbosity::OBNOXIOUS)
+
+    includes = 
+      @line_marker_includes_extractor.extract_includes_from_file(
+        preprocessed_filepath,
+        PreprocessinatorLineMarkerIncludesExtractor::SYSTEM,
+        5 # Practical max depth limit for system headers (to avoid noisy length)
+      )
+
+    return clean_self_reference( filepath, includes )
+  end
+
+  def extract_system_includes_from_text(name:, filepath:, defines: [])
+    includes = []
+
+    filename = File.basename(filepath)
+
+    msg = @reportinator.generate_module_progress(
+      operation: "Extracting system #includes from original file using fallback method",
+      module_name: name,
+      filename: filename
+    )
+    @loginator.log( msg, Verbosity::OBNOXIOUS, LogLabels::WARNING )
+
+    cond_tracker = CPreprocessorConditionals.new( defines )
+
+    # Open in binary mode: code_lines applies clean_encoding per-line, but each_line
+    # itself can raise on invalid byte sequences before clean_encoding is reached.
+    @file_wrapper.open(filepath, 'rb') do |input|
+      @parsing_parcels.code_lines( input ) do |line|
+        cond_tracker.process_directive( line )
+        next unless cond_tracker.active?
+        _include = @include_factory.system_include_from_directive( line )
+        includes << _include if !_include.nil?
+      end
+    end
+
+    return clean_self_reference( filepath, includes )
+  end
+
+  # Write to disk a yaml representation of a list of includes
+  def write_includes_list(filepath, list)
+    @yaml_wrapper.dump(filepath, Includes.to_hashes(list))
+  end
+
+  def load_includes_list(filepath)
+    loaded = begin
+      @yaml_wrapper.load( filepath )
+    rescue YamlLoadException => e
+      raise YamlLoadException.new(
+        reason: e.reason, source: e.source, original_error: e.original_error,
+        message: "Cached #include list is corrupted or unreadable ⏩️ #{e.message}"
+      )
+    end
+
+    # Note: It's possible empty YAML content returns nil so ensure empty list
+    return Includes.from_hashes( loaded || [] )
+  end
+
+  ### Private ###
+  private
+
+  # Remove any filepath in the includes list that is identical to the filepath being processed.
+  # We want to prevent an includes list containing an unnecessary self-reference.
+  # Use normalized paths for comparison to handle variations (relative vs absolute, different separators, etc.)
+  def clean_self_reference(filepath, includes)
+    _filepath = File.expand_path(filepath)
+    Includes.sanitize!(includes) do |include, _|
+      _filepath == File.expand_path(include.filepath)
+    end
+    return includes
+  end
+
+end

data/lib/ceedling/preprocess/preprocessinator_line_marker_includes_extractor.rb

@@ -0,0 +1,215 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/exceptions'
+require 'set'
+
+##
+## System Includes Directives-only Preprocessor Output Parsing
+## ===========================================================
+##
+## Format:
+##  - File content excerpted between line markers nesting the expansion of files.
+##  - Line marker: `# linenum filename flags`
+##  - Flags:
+##     - 1 = Start of new file
+##     - 2 = Returning to a file
+##     - 3 = System header
+##     - 4 = Implicit extern C
+##
+## Notes:
+##  - The initial line marker is "# 1" followed by the source file being preprocessed
+##  - `# 0` at the top of the output is reserved for internal macros, command line macros
+##    and other symbols not from #include directives.
+##  - Because of the preprocessor's behavior around #include guards, the order of #include
+##    directives can mask the header filenames. That is, in the following examples,
+##    <stdint.h> is at depth two.
+##
+##  #include "ecmsi_bar.h"
+##  #include "ecmsi_foo.h" // Includes <stdint.h>
+##  #include <stdint.h>    // Does not appear in preprocessor output.
+##                         // Transitive <stdint.h> from preceding user include at depth 2.
+##
+## Example output follows
+## (Edited for length)
+## -----------------------------------------------------------------------------------------
+## # 0 "<command-line>" 2
+## # 1 "src/external_calls_multi_static_inline/ecmsi_bar.c"
+## 
+## /****************************************************************************************
+##  * Includes
+##  ***************************************************************************************/
+## # 1 "src/external_calls_multi_static_inline/ecmsi_bar.h" 1
+## 
+## #define ECMSI_BAR_H 
+## 
+## /****************************************************************************************
+##  * Public function prototypes
+##  ***************************************************************************************/
+## extern void ecmsi_bar_init(void);
+## 
+## # 6 "src/external_calls_multi_static_inline/ecmsi_bar.c" 2
+## # 1 "src/external_calls_multi_static_inline/ecmsi_foo.h" 1
+## 
+## #define ECMSI_FOO_H 
+## 
+## /****************************************************************************************
+##  * Includes
+##  ***************************************************************************************/
+## # 1 "/usr/lib/gcc/x86_64-linux-gnu/12/include/stdint.h" 1 3 4
+## 
+## 
+## # 1 "/usr/include/stdint.h" 1 3 4
+## /* Copyright (C) 1997-2022 Free Software Foundation, Inc.
+##    This file is part of the GNU C Library.
+## */
+## 
+## #define _STDINT_H 1
+## 
+## #define __GLIBC_INTERNAL_STARTING_HEADER_IMPLEMENTATION 
+## 
+## # 318 "/usr/include/stdint.h" 3 4
+## 
+## # 10 "/usr/lib/gcc/x86_64-linux-gnu/12/include/stdint.h" 2 3 4
+## 
+## #define _GCC_WRAP_STDINT_H 
+## # 9 "src/external_calls_multi_static_inline/ecmsi_foo.h" 2
+##
+
+# Parse GCC preprocessor output (from -fdirectives-only) to extract system include directives
+class PreprocessinatorLineMarkerIncludesExtractor
+  LINE_MARKER_REGEX = /^#\s+(\d+)\s+"([^"]+)"(?:\s+(\d+(?:\s+\d+)*))?$/ unless const_defined?(:LINE_MARKER_REGEX)
+
+  SYSTEM = :system  unless const_defined?(:SYSTEM)
+  USER   = :user    unless const_defined?(:USER)
+
+  constructor :include_factory
+
+  # Parse preprocessor output from a file (production use)
+  # @param filepath [String] Path to the preprocessor output file
+  # @return [Array<UserInclude, SystemInclude>]
+  def extract_includes_from_file(filepath, type, max_depth=nil)
+    validate_type_argument( type )
+    includes = []
+    begin
+      # Open in binary mode: GCC output under a non-C locale contains non-ASCII bytes
+      # (e.g. <組み込み> for <built-in> under ja_JP). Text mode would interpret those
+      # bytes using the locale-dependent external encoding (e.g. Windows-31J on ja_JP
+      # Windows), raising an encoding error on read. Binary mode bypasses that translation.
+      # LINE_MARKER_REGEX uses only ASCII delimiters and is safe in binary mode.
+      # The filepath.start_with?('<') guard correctly skips localized markers because
+      # their first byte is 0x3C — ASCII '<' — regardless of the surrounding encoding.
+      # NOTE: binary mode means \r\n line endings are NOT translated on Windows; the
+      # extract_includes method calls line.chomp! before regex matching to handle this.
+      File.open(filepath, 'rb') do |file|
+        includes = extract_includes(io: file, filepath: filepath, type: type, max_depth: max_depth)
+      end
+    rescue StandardError => e
+      raise CeedlingException.new("Failed to extract #{type} includes from preprocessor output file '#{filepath}' ⏩️ #{e.message}")
+    end
+    return includes
+  end
+
+  # Parse preprocessor output from a string (testing use)
+  # @param content [String] Preprocessor output as a string
+  # @return [Array<UserInclude, SystemInclude>]
+  def extract_includes_from_string(content, filepath, type, max_depth=nil)
+    validate_type_argument( type )
+    require 'stringio'
+    io = StringIO.new(content)
+    return extract_includes(io: io, filepath: filepath, type: type, max_depth: max_depth)
+  end
+
+  private
+
+  def validate_type_argument(type)
+    unless [SYSTEM, USER].include?(type)
+      raise CeedlingException.new("Invalid type argument: #{type.inspect}. Must be :#{SYSTEM} or :#{USER}")
+    end
+  end
+
+  # Extracts includes from directives-only preprocessor output
+  # Returns an array of Include-derived objects
+  def extract_includes(io:, filepath:, type:, max_depth:)
+    includes = []
+    seen_paths = Set.new
+    initial_file_seen = false
+    depth = 0
+
+    # Extract just the filename from full path
+    source_filename = File.basename(filepath)
+    
+    io.each_line do |line|
+      # Binary mode ('rb') yields raw bytes without platform newline translation.
+      # On Windows (MinGW/MSYS2), GCC writes \r\n line endings in preprocessor output.
+      # IO#each_line splits on \n and preserves the preceding \r in the yielded string.
+      # LINE_MARKER_REGEX ends with $ which anchors before \n but does NOT match when
+      # \r immediately precedes it — the optional flags group (?:\s+(\d+...))? cannot
+      # consume the stray \r, so every line marker match fails on Windows, producing
+      # an empty include list and cascading test build failures.
+      # String#chomp! (no-arg) removes \r\n, \r, or \n — safe on all platforms.
+      line.chomp!
+
+      # Match GCC line markers
+      if (match = LINE_MARKER_REGEX.match(line))
+        # String filename
+        filepath = match[2]
+
+        # Skip special markers like "<built-in>" and "<command-line>"
+        next if filepath.start_with?('<')
+
+        # Integer line number
+        line_number = match[1].to_i
+        
+        # Array of flag integers
+        flags = match[3] ? match[3].split.map(&:to_i) : []
+        
+        # Look for `# 1 "<filename>"`
+        if !initial_file_seen
+          if (line_number == 1) && (File.basename(filepath) == source_filename)
+            initial_file_seen = true
+            depth = 1
+          end
+          next
+        end
+        
+        # Flag 1 means entering a new file
+        if flags.include?(1)
+          depth += 1
+
+          # Skip if we've already seen this path
+          next if seen_paths.include?( filepath )
+
+          # Skip if max depth is defined and we've exceeded it.
+          # If max depth is not defined, do not limit the depth.
+          next if (max_depth && (depth > max_depth))
+
+          seen_paths.add(filepath)
+
+          # Extract system includes
+          if type == SYSTEM
+            # Flag 3 indicates a system header
+            if flags.include?(3)
+              includes << @include_factory.system_include_from_filepath( filepath )
+            end
+          # Extract user includes
+          elsif type == USER
+            unless flags.include?(3)
+              includes << @include_factory.user_include_from_filepath( filepath )
+            end
+          end
+        # Flag 2 means returning to a previous file
+        elsif flags.include?(2)
+          depth -= 1 if depth > 0
+        end
+      end
+    end
+    
+    return includes
+  end
+end
+