ceedling 1.0.1 → 1.1.0

data/lib/ceedling/preprocess/preprocessinator_code_finder.rb

@@ -0,0 +1,150 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'stringio'
+
+class PreprocessinatorCodeFinder
+
+  LINE_MARKER_REGEX = /^#\s+(\d+)\s+"[^"]+"[^\n]*\n/ unless const_defined?(:LINE_MARKER_REGEX)
+
+  # Regex-special-character-immune string
+  WHITESPACE_MARKER = '<!@_ws_!@>' unless const_defined?(:WHITESPACE_MARKER)
+
+  # Open a GCC preprocessor output file and search it for code.
+  # Returns the 1-indexed source line number of the match, or nil if not found.
+  # Intended for production use where preprocessor output resides on disk.
+  def find_in_preprpocessed_file(filepath, code)
+    File.open( filepath, 'r' ) do |file|
+      return find_in_preprocessed_content( io: file, search: code )
+    end
+  end
+
+
+  # Wrap a GCC preprocessor output string in a StringIO and search it for code.
+  # Returns the 1-indexed source line number of the match, or nil if not found.
+  # Intended for test use so that specs require no temporary files.
+  def find_in_preprpocessed_string(content, code)
+    buffer = StringIO.new( content )
+    return find_in_preprocessed_content( io: buffer, search: code )
+  end
+
+  # Open a C source file and search it for code.
+  # Returns the 1-indexed source line number of the match, or nil if not found.
+  # Intended for production use where C file resides on disk.
+  def find_in_c_file(filepath, code)
+    File.open( filepath, 'r' ) do |file|
+      return find_in_c_code( io: file, search: code )
+    end
+  end
+
+
+  # Wrap a C file content string in a StringIO and search it for code.
+  # Returns the 1-indexed source line number of the match, or nil if not found.
+  # Intended for test use so that specs require no temporary files.
+  def find_in_c_string(content, code)
+    buffer = StringIO.new( content )
+    return find_in_c_code( io: buffer, search: code )
+  end
+
+  private
+
+  # Search a GCC preprocessor output IO stream for an exact match of search.
+  #
+  # GCC preprocessor output intersperses the expanded source text with line
+  # markers of the form:
+  #   # <linenum> "<filename>" [flags]
+  #
+  # Each marker declares that the source line immediately following it
+  # corresponds to line linenum in the named file. Subsequent non-marker lines
+  # increment the source line count by one each.
+  #
+  # The method locates `search`` as a substring of the full stream content, then
+  # walks backwards through the line markers that precede the match to find the
+  # most recent one. The source line number is computed as:
+  #   last_marker_linenum + (newlines between marker end and match start)
+  #
+  # Returns nil when search is not present in the stream or no line marker
+  # precedes the match (which indicates malformed preprocessor output).
+  def find_in_preprocessed_content(io:, search:)
+    content = io.read
+    return nil if content.nil? || content.empty?
+
+    # Locate the search string within the preprocessor output
+    match_pos = whitespace_insensitive_search( content, search )
+    return nil if match_pos.nil?
+
+    # Examine only the content before the match for GCC line markers.
+    # Line markers have the form: # <linenum> "<filename>" [optional flags]
+    # The linenum in each marker refers to the source line immediately following it.
+    prefix = content[0, match_pos]
+
+    last_marker_num = nil
+    # Byte position in content after the last marker's newline
+    last_marker_end = 0
+
+    prefix.scan( LINE_MARKER_REGEX ) do |captures|
+      last_marker_num = captures[0].to_i
+      last_marker_end = $~.end(0)
+    end
+
+    if last_marker_num.nil?
+      # No line marker precedes the match -- something is wrong
+      return nil
+    end
+
+    # Each newline between the end of the last line marker line and the match start
+    # advances the source line by one. The marker's linenum is the base.
+    newlines_after_marker = content[last_marker_end, match_pos - last_marker_end].count("\n")
+    
+    # Return line number of found code (mostly for test validation)
+    return last_marker_num + newlines_after_marker
+  end
+
+  # Search a C code IO stream for an exact match of search.
+  #
+  # The method locates `search`` as a substring of the full stream content, then
+  # identifies the source line number. Returns nil when search is not present in 
+  # the stream.
+  def find_in_c_code(io:, search:)
+    content = io.read
+    return nil if content.nil? || content.empty?
+
+    # Locate the search string within the preprocessor output
+    match_pos = whitespace_insensitive_search( content, search )
+    return nil if match_pos.nil?
+
+    return (content[0, match_pos].count("\n")) + 1
+  end
+
+  private
+
+  def whitespace_insensitive_search(content, code)
+    # Collapse whitespace to a unique token
+    _code = code.gsub(/\s+/, WHITESPACE_MARKER)
+
+    # Escape the code block to search for.
+    # Note: Regexp#escape escapes each piece of whitespace and, so, we must take the preparation step above.
+    _code = Regexp.escape( _code )
+
+    # Replace the whitespace token with regex whitespace-insenstive matcher
+    _code.gsub!(/#{WHITESPACE_MARKER}/, '\s+')
+
+    # Create the regex we'll use
+    pattern = Regexp.new( _code )
+
+    # Try to find the code block
+    matched = pattern.match( content )
+
+    # Failed to match
+    return matched if matched.nil?
+
+    # Return position of the match within `content`
+    return matched.begin( 0 )
+  end
+
+
+end

data/lib/ceedling/preprocess/preprocessinator_comment_stripper.rb

@@ -0,0 +1,80 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'stringio'
+require 'ceedling/exceptions'
+
+class PreprocessinatorCommentStripper
+
+  constructor :c_comment_scanner
+
+
+  # Strip all C comments from a file. The file is unchanged if no comments 
+  # are found. The routine returns `true` if changes are made, `false` otherwise.
+  #
+  # Every single-line comment is replaced by a single space. Every multi-line
+  # comment is replaced by an equivalent number of newlines.
+  def strip_file(filepath)
+    stripped = nil
+
+    begin
+      # Open in binary mode to avoid locale-dependent encoding failures.
+      # GCC preprocessor output may contain localized strings (e.g. <組み込み> under ja_JP locale).
+      # CCommentScanner already operates byte-accurately internally.
+      File.open(filepath, 'rb') do |buffer|
+        stripped = strip(buffer)
+      end
+    rescue => e
+      raise CeedlingException.new("Failed to read '#{filepath}' for comment stripping ⏩️ #{e}")
+    end
+
+    # No change
+    return false if stripped.nil?
+
+    begin
+      # Write in binary mode to match binary read — preserves original line endings exactly
+      File.write(filepath, stripped, mode: 'wb')
+    rescue => e
+      raise CeedlingException.new("Failed to rewrite '#{filepath}' after comment stripping ⏩️ #{e}")
+    end
+
+    return true
+  end
+
+  # Strip all C comments from a string content and return the cleaned content 
+  # as a String. The string is unchanged if no comments are found.
+  #
+  # Every single-line comment is replaced by a single space. Every multi-line
+  # comment is replaced by an equivalent number of newlines.
+  def strip_string(content)
+    buffer = StringIO.new(content)
+    stripped = strip(buffer)
+
+    return content if stripped.nil?
+
+    return stripped
+  end
+
+  private
+
+  def strip(buffer)
+    # Search buffer for comments
+    comment_infos = @c_comment_scanner.scan(io: buffer)
+    return nil if comment_infos.empty?
+
+    # Reset buffer
+    buffer.rewind
+
+    # Remove comments from content of buffer
+    return @c_comment_scanner.remove(
+      buffer.read,
+      comment_infos,
+      mode: CCommentScanner::PRESERVE_LINES
+    )
+  end
+
+end

data/lib/ceedling/preprocess/preprocessinator_file_assembler.rb

@@ -0,0 +1,378 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'rake' # for ext() method
+require 'ceedling/file_wrapper'
+require 'ceedling/encodinator'
+require 'ceedling/preprocess/c_preprocessor_conditionals'
+
+class PreprocessinatorFileAssembler
+
+  constructor(
+    :preprocessinator_reconstructor,
+    :configurator,
+    :tool_executor,
+    :file_path_utils,
+    :file_wrapper,
+    :parsing_parcels,
+    :loginator,
+    :reportinator
+  )
+
+  def collect_mockable_header_file_contents(
+      test:,
+      filepath:,
+      directives_only_filepath:,
+      fallback:,
+      flags:,
+      defines:,
+      include_paths:,
+      extras:
+    )
+    contents = []
+
+    # Our extra file content to be preserved
+    # Leave these empty if :extras is false
+    pragmas = []
+    macro_defs = []
+
+    preprocessed_filepath = @file_path_utils.form_preprocessed_file_full_expansion_filepath( filepath, test )
+
+    # Run GCC with full preprocessor expansion
+    command = @tool_executor.build_command_line(
+      @configurator.tools_test_file_full_preprocessor,
+      # Additional arguments
+      flags,
+      # Argument replacement
+      filepath,
+      preprocessed_filepath,
+      defines,
+      include_paths
+    )    
+    @tool_executor.exec( command )
+
+    @file_wrapper.open( preprocessed_filepath, 'r' ) do |file|
+      contents = @preprocessinator_reconstructor.extract_file_as_array_from_expansion( file, filepath )
+    end
+
+    # Bail out if no extras are required
+    return contents, [] if !extras
+
+    # Try to find an #include guard in the first 2k of the file text.
+    # An #include guard is one macro from the original file we don't want to preserve if we can help it.
+    # We create our own #include guard in the header file we create.
+    # It's possible preserving the macro from the original file's #include guard could trip something up.
+    # Of course, it's also possible some header conditional compilation feature is dependent on it.
+    # ¯\_(ツ)_/¯
+    include_guard = @preprocessinator_reconstructor.extract_include_guard( @file_wrapper.read( filepath, 2048 ).clean_encoding )
+
+    if fallback
+      msg = @reportinator.generate_module_progress(
+        operation: "Using fallback method to extract pragmas and macros from",
+        module_name: test,
+        filename: File.basename(filepath)
+      )
+      @loginator.log( msg, Verbosity::OBNOXIOUS, LogLabels::WARNING )
+
+      # Open in binary mode + clean encoding: source files may contain non-ASCII bytes
+      # in comments (e.g. © symbols) that cause encoding errors under non-C locale.
+      @file_wrapper.open( filepath, 'rb' ) do |file|
+        # TODO: Modify to process line-at-a-time for memory savings & performance boost
+        _contents = file.read.clean_encoding
+
+        # Filter out inactive conditional blocks before extraction so that only
+        # pragmas and macros that are active for the current defines list are returned.
+        _contents = _filter_conditionals( _contents, defines )
+
+        # Extract pragmas and macros from
+        pragmas = @preprocessinator_reconstructor.extract_pragmas( _contents )
+        macro_defs = @preprocessinator_reconstructor.extract_macro_defs( _contents, include_guard )
+      end
+    else
+      @file_wrapper.open( directives_only_filepath, 'r' ) do |file|
+        # Get code contents of preprocessed directives-only file as a string
+        # TODO: Modify to process line-at-a-time for memory savings & performance boost
+        _contents = @preprocessinator_reconstructor.extract_file_as_string_from_expansion( file, filepath )
+
+        # Extract pragmas and macros from 
+        pragmas = @preprocessinator_reconstructor.extract_pragmas( _contents )
+        macro_defs = @preprocessinator_reconstructor.extract_macro_defs( _contents, include_guard )
+      end
+    end
+
+    return contents, (pragmas + macro_defs)
+  end
+
+
+  def collect_file_contents_from_directives_only_preprocessing(source_filepath:, test:)
+    contents = []
+
+    preprocessed_filepath = @file_path_utils.form_preprocessed_file_raw_directives_only_filepath( source_filepath, test )
+
+    @file_wrapper.open( preprocessed_filepath, 'r' ) do |file|
+      contents = @preprocessinator_reconstructor.extract_file_as_array_from_expansion( file, source_filepath )
+    end
+
+    return contents
+  end
+
+  # Extract macro definitions and pragmas from a file using text scanning (fallback mode).
+  # Used to preserve #define macros and #pragma directives in partial file reconstruction
+  # when the directives-only preprocessor (-fdirectives-only) is not available.
+  # Mirrors the fallback branch of collect_mockable_header_file_contents.
+  def collect_macros_and_pragmas_fallback(source_filepath:, defines: [])
+    include_guard = @preprocessinator_reconstructor.extract_include_guard(
+      @file_wrapper.read( source_filepath, 2048 ).clean_encoding
+    )
+
+    pragmas = []
+    macro_defs = []
+
+    @file_wrapper.open( source_filepath, 'rb' ) do |file|
+      _contents = file.read.clean_encoding
+      _contents = _filter_conditionals( _contents, defines )
+      pragmas    = @preprocessinator_reconstructor.extract_pragmas( _contents )
+      macro_defs = @preprocessinator_reconstructor.extract_macro_defs( _contents, include_guard )
+    end
+
+    return pragmas + macro_defs
+  end
+
+
+  def collect_file_contents_fallback(source_filepath:, defines: [])
+    # Open in binary mode + clean encoding: source files may have non-ASCII bytes
+    # in comments (e.g. © symbols) that raise encoding errors under locale-dependent
+    # text-mode encoding on non-C-locale systems.
+    # Call line.chomp before clean_encoding: readlines(chomp:true) in binary mode
+    # removes the \n separator but leaves \r on Windows CRLF lines. clean_encoding's
+    # :universal_newline option would then convert that lone \r → \n, producing lines
+    # with a spurious trailing \n that causes double-newlines in assembled output.
+    # String#chomp (no-arg) removes \r\n, \r, or \n — safe on all platforms.
+    cond_tracker = CPreprocessorConditionals.new( defines )
+    result = []
+
+    @file_wrapper.open( source_filepath, 'rb' ) do |file|
+      file.readlines( chomp: true ).each do |raw_line|
+        line = raw_line.chomp.clean_encoding
+
+        # Strip inline // comment from a temporary copy for directive detection only.
+        # We preserve the original line (with comments) in output per design intent.
+        stripped = line.sub( /\s*\/\/.*$/, '' ).lstrip
+
+        # Feed the comment-stripped line to the conditional tracker.
+        # The tracker only reacts to lines beginning with '#'.
+        cond_tracker.process_directive( stripped )
+
+        # Skip all preprocessor directive lines from output (they appear in the
+        # assembled file via the includes list and extras rather than raw content).
+        # The non-fallback path produces GCC expansion output which contains no
+        # literal #include/#define/#pragma lines either.
+        next if stripped.start_with?( '#' )
+
+        # Skip lines inside inactive conditional blocks
+        next unless cond_tracker.active?
+
+        result << line
+      end
+    end
+
+    return result
+  end
+
+
+  def collect_file_contents_from_full_expansion(source_filepath:, test:)
+    contents = []
+
+    full_expansion_filepath = @file_path_utils.form_preprocessed_file_full_expansion_filepath( source_filepath, test )
+
+    @file_wrapper.open( full_expansion_filepath, 'r' ) do |file|
+      contents = @preprocessinator_reconstructor.extract_file_as_array_from_expansion( file, source_filepath )
+    end
+
+    return contents
+  end
+
+
+  def assemble_preprocessed_header_file(filename:, preprocessed_filepath:, contents:, extras:, includes:)
+    # Generate #include guard name for header files
+    guardname = FileWrapper.generate_include_guard( filename )
+
+    # Write contents of final preprocessed file a line at a time
+    # ----------------------------------------------------------
+    @file_wrapper.open( preprocessed_filepath, 'w' ) do |file|
+      # Add include guards and extra blank lines to beginning of file contents
+      file << "#ifndef #{guardname}\n"
+      file << "#define #{guardname}\n\n"
+
+      # Reinsert #include statements into stripped down file
+      # Rely on Include object stringification for formatting of incudes
+      includes.each { |include| file << "#{include}\n" }
+
+      # Blank line
+      file << "\n" unless includes.empty?
+
+      # Add in any macro defintions or prgamas
+      extras.each do |ex|
+        if ex.class == String
+          file << ex + "\n"
+
+        elsif ex.class == Array
+          ex.each { |line| file << line + "\n" }
+        end
+
+        # Blank line
+        file << "\n"
+      end
+
+      # Add extracted contents from preprocessed file
+      contents.each { |line| file << line + "\n" }
+
+      # Add final rear guard with extra blank lines
+      file << "\n#endif // #{guardname}\n"
+    end
+  end
+
+
+  def collect_test_file_contents(
+      test:,
+      filepath:,
+      directives_only_filepath:,
+      fallback:,
+      flags:,
+      defines:,
+      include_paths:
+    )
+    contents = []
+    # TEST_SOURCE_FILE() and TEST_INCLUDE_PATH()
+    test_directives = []
+    # TEST_CASE() / TEST_RANGE() / TEST_MATRIX(), paired with the test function name each precedes
+    test_case_directives = []
+
+    preprocessed_filepath = @file_path_utils.form_preprocessed_file_full_expansion_filepath( filepath, test )
+
+    # Run GCC with full preprocessor expansion
+    command = @tool_executor.build_command_line(
+      @configurator.tools_test_file_full_preprocessor,
+      # Additional arguments
+      flags,
+      # Argument replacement
+      filepath,
+      preprocessed_filepath,
+      defines,
+      include_paths
+    )
+    @tool_executor.exec( command )
+
+    @file_wrapper.open( preprocessed_filepath, 'r' ) do |file|
+      contents = @preprocessinator_reconstructor.extract_file_as_array_from_expansion( file, filepath )
+    end
+
+    if fallback
+      msg = @reportinator.generate_module_progress(
+        operation: "Using fallback method to extract test directive macros from",
+        module_name: test,
+        filename: File.basename(filepath)
+      )
+      @loginator.log( msg, Verbosity::OBNOXIOUS, LogLabels::WARNING )
+
+      # Open in binary mode + clean encoding: source files may contain non-ASCII bytes
+      # in comments (e.g. © symbols) that cause encoding errors under non-C locale.
+      @file_wrapper.open( filepath, 'rb' ) do |file|
+        _contents = file.read.clean_encoding
+
+        # Filter out inactive conditional blocks before extraction so that only
+        # test directives that are active for the current defines list are returned.
+        _contents = _filter_conditionals( _contents, defines )
+
+        # Extract TEST_SOURCE_FILE() and TEST_INCLUDE_PATH()
+        test_directives = @preprocessinator_reconstructor.extract_test_directive_macro_calls( _contents )
+
+        # Extract TEST_CASE()/TEST_RANGE()/TEST_MATRIX() calls paired with the test function
+        # they immediately precede -- these vanish from `contents` above because they're real
+        # (empty-expanding) Unity macros erased by full preprocessor expansion, so we must
+        # recover them from this macro-preserving text instead.
+        test_case_directives = @preprocessinator_reconstructor.extract_test_case_directives( _contents )
+      end
+    else
+      @file_wrapper.open( directives_only_filepath, 'r' ) do |file|
+        # Get code contents of preprocessed directives-only file as a string
+        _contents = @preprocessinator_reconstructor.extract_file_as_string_from_expansion( file, filepath )
+
+        # Extract TEST_SOURCE_FILE() and TEST_INCLUDE_PATH()
+        test_directives = @preprocessinator_reconstructor.extract_test_directive_macro_calls( _contents )
+
+        # Extract TEST_CASE()/TEST_RANGE()/TEST_MATRIX() calls paired with the test function
+        # they immediately precede (see comment in the fallback branch above)
+        test_case_directives = @preprocessinator_reconstructor.extract_test_case_directives( _contents )
+      end
+    end
+
+    # Reinsert TEST_CASE()/TEST_RANGE()/TEST_MATRIX() calls immediately ahead of their test
+    # function in `contents` -- full expansion erased them in place with no trace (not even a
+    # blank line), so position must be recovered by matching on the function name they modify.
+    contents = @preprocessinator_reconstructor.splice_test_case_directives(
+      contents: contents,
+      directives: test_case_directives
+    )
+
+    return contents, test_directives
+  end
+
+
+  ### Private ###
+
+  private
+
+  # Filters `file_contents` string through CPreprocessorConditionals, returning
+  # only lines in active conditional blocks. Uses code_lines internally so
+  # comments are stripped and line continuations are joined before filtering.
+  # Intended for token extraction (pragmas, macro defs, test directives) — NOT
+  # for content output where comments must be preserved.
+  def _filter_conditionals(file_contents, defines)
+    tracker = CPreprocessorConditionals.new( defines )
+    lines = []
+    @parsing_parcels.code_lines( file_contents ) do |line|
+      tracker.process_directive( line )
+      lines << line if tracker.active?
+    end
+    lines.join("\n") + "\n"
+  end
+
+  public
+
+  def assemble_preprocessed_code_file(filename:, preprocessed_filepath:, contents:, extras:, includes:)
+    # Write contents of final preprocessed file a line at a time
+    # ----------------------------------------------------------
+    @file_wrapper.open( preprocessed_filepath, 'w' ) do |file|
+      # Reinsert #include statements into stripped down file
+      # Rely on Include object stringification for formatting of incudes
+      includes.each { |include| file << "#{include}\n" }
+
+      # Blank line
+      file << "\n" unless includes.empty?
+
+      # Add in any extras like test directive macros or preserved macro definitions
+      extras.each do |ex|
+        if ex.class == String
+          file << ex + "\n"
+        elsif ex.class == Array
+          ex.each { |line| file << line + "\n" }
+        end
+      end
+
+      # Blank line
+      file << "\n" unless extras.empty?
+
+      # Add extracted contents from preprocessed file
+      contents.each { |line| file << line + "\n" }
+
+      # Blank line
+      file << "\n" unless contents.empty?
+    end
+  end
+
+end