i18n-context-generator 0.3.0 → 0.5.0

data/lib/i18n_context_generator/searcher/match_filtering.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  class Searcher
+    # Removes duplicate, translation-definition, and non-localization matches.
+    module MatchFiltering
+      private
+
+      def filter_matches(matches, key)
+        seen = Set.new
+        explicit_patterns = explicit_localization_patterns(key)
+        matches.select do |match|
+          location = "#{match.file}:#{match.line}"
+          next false if seen.include?(location)
+          next false if false_positive?(match.match_line, key, explicit_patterns)
+          next false if translation_file?(match.file)
+
+          seen.add(location)
+        end
+      end
+
+      def false_positive?(line, key, explicit_patterns)
+        return false if line.nil? || line.empty?
+        return false if explicit_localization_usage?(line, explicit_patterns)
+
+        quoted_key = "[\"']#{Regexp.escape(key)}[\"']"
+        comparison_patterns = [
+          /==\s*#{quoted_key}/,
+          /#{quoted_key}\s*==/,
+          /!=\s*#{quoted_key}/,
+          /#{quoted_key}\s*!=/,
+          /\.equals\(\s*#{quoted_key}/,
+          /contentEquals\(\s*#{quoted_key}/
+        ]
+
+        comparison_patterns.any? { |pattern| pattern.match?(line) }
+      end
+
+      def explicit_localization_patterns(key)
+        (build_ios_patterns(key) + build_android_patterns(key)).map { |pattern| Regexp.new(pattern) }
+      end
+
+      def explicit_localization_usage?(line, patterns)
+        patterns.any? { |pattern| pattern.match?(line) }
+      end
+
+      def translation_file?(file)
+        !FileClassifier.translation_platform(file).nil?
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/searcher/source_discovery.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  class Searcher
+    # Source-discovery helpers used for source-first extraction runs.
+    module SourceDiscovery
+      IOS_COMMENT_ARGUMENT_PATTERN = /\bcomment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["']/
+
+      def discover_localization_entries
+        entries = discover_files.flat_map do |file|
+          discover_entries_in_file(file)
+        end
+
+        deduplicate_discovered_entries(entries)
+      end
+
+      private
+
+      def discover_entries_in_file(file)
+        case platform_for_file(file)
+        when :ios
+          discover_ios_entries(file)
+        when :android
+          discover_android_entries(file)
+        else
+          []
+        end
+      rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR => e
+        warn "Warning: Could not read #{file}: #{e.message}" if $VERBOSE
+        []
+      rescue ArgumentError => e
+        return [] if e.message.include?('invalid byte sequence')
+
+        raise
+      end
+
+      def deduplicate_discovered_entries(entries)
+        entries.each_with_object({}) do |entry, deduplicated_entries|
+          identity = [entry.resource_type, entry.key]
+          existing_entry = deduplicated_entries[identity]
+          if existing_entry.nil?
+            deduplicated_entries[identity] = entry
+            next
+          end
+
+          preferred_entry = if existing_entry.comment.to_s.empty? && !entry.comment.to_s.empty?
+                              entry
+                            else
+                              existing_entry
+                            end
+          deduplicated_entries[identity] = DiscoveredLocalization.new(
+            key: preferred_entry.key,
+            file: preferred_entry.file,
+            line: preferred_entry.line,
+            text: preferred_entry.text,
+            comment: preferred_entry.comment,
+            resource_type: preferred_entry.resource_type,
+            locations: (existing_entry.locations + entry.locations).uniq,
+            location_groups: (existing_entry.location_groups + entry.location_groups).uniq
+          )
+        end.values
+      end
+
+      def platform_for_file(file)
+        FileClassifier.searchable_platform(file, platform_hint: @platform)
+      end
+
+      def discover_ios_entries(file)
+        lines = searchable_file_lines(file)
+        entries = []
+        index = 0
+
+        while index < lines.length
+          line = lines[index]
+          next_index, discovered_entry = extract_ios_entry(lines, file, index, line)
+          entries << discovered_entry if discovered_entry
+          index = next_index || (index + 1)
+        end
+
+        entries
+      end
+
+      def extract_ios_entry(lines, file, index, line)
+        if @localization_syntax.ios_call_start_patterns.any? { |pattern| pattern.match?(line) }
+          next_index, discovered_entry = extract_ios_multiline_entry(lines, file, index)
+          return [next_index, discovered_entry] if discovered_entry
+        end
+
+        if (entry = extract_ios_single_line_entry(file, index, line))
+          return [index + 1, entry]
+        end
+
+        [index + 1, nil]
+      end
+
+      def extract_ios_single_line_entry(file, index, line)
+        pattern = @localization_syntax.ios_single_line_discovery_patterns.find { |candidate| candidate.match?(line) }
+        return unless pattern
+
+        build_ios_discovered_entry(
+          file,
+          index,
+          pattern.match(line),
+          comment_match: IOS_COMMENT_ARGUMENT_PATTERN.match(line)
+        )
+      end
+
+      def extract_ios_multiline_entry(lines, file, start_index, lookahead: 8)
+        end_index = ios_call_end_index(lines, start_index, lookahead: lookahead)
+        snippet = lines[start_index..end_index].join("\n")
+        pattern = @localization_syntax.ios_multiline_discovery_patterns.find { |candidate| candidate.match?(snippet) }
+        return [start_index + 1, nil] unless pattern
+
+        match = pattern.match(snippet)
+        key_line_index = locate_key_line(lines, start_index, end_index, match[:key])
+        comment_match = IOS_COMMENT_ARGUMENT_PATTERN.match(snippet)
+        locations = (start_index..end_index).map { |index| "#{file}:#{index + 1}" }
+
+        [
+          end_index + 1,
+          build_ios_discovered_entry(
+            file,
+            key_line_index || start_index,
+            match,
+            comment_match: comment_match,
+            locations: locations
+          )
+        ]
+      end
+
+      def build_ios_discovered_entry(file, index, match, comment_match: nil, locations: nil)
+        key = unescape_source_string(match[:key])
+        return if key.nil? || key.empty?
+
+        text = match.names.include?('text') ? match[:text] : nil
+        comment = if comment_match
+                    unescape_source_string(comment_match[:comment])
+                  elsif match.names.include?('comment')
+                    unescape_source_string(match[:comment])
+                  end
+        text = unescape_source_string(text) if text
+
+        DiscoveredLocalization.new(
+          key: key,
+          file: file,
+          line: index + 1,
+          text: text,
+          comment: comment,
+          locations: locations
+        )
+      end
+
+      def ios_call_end_index(lines, start_index, lookahead:)
+        maximum_index = [lines.length - 1, start_index + lookahead].min
+        depth = 0
+        found_opening = false
+
+        (start_index..maximum_index).each do |index|
+          parenthesis_delta(lines[index]).each do |delta|
+            found_opening = true if delta.positive?
+            depth += delta
+          end
+          return index if found_opening && depth <= 0
+        end
+
+        maximum_index
+      end
+
+      def parenthesis_delta(line)
+        deltas = []
+        quote = nil
+        escaped = false
+
+        line.each_char do |character|
+          if quote
+            if escaped
+              escaped = false
+            elsif character == '\\'
+              escaped = true
+            elsif character == quote
+              quote = nil
+            end
+          elsif ["'", '"'].include?(character)
+            quote = character
+          elsif character == '('
+            deltas << 1
+          elsif character == ')'
+            deltas << -1
+          end
+        end
+        deltas
+      end
+
+      def locate_key_line(lines, start_index, end_index, key)
+        (start_index..end_index).find do |index|
+          lines[index].include?("\"#{key}\"") || lines[index].include?("@\"#{key}\"")
+        end
+      end
+
+      def discover_android_entries(file)
+        lines = searchable_file_lines(file)
+        entries = []
+
+        lines.each_with_index do |line, index|
+          entries.concat(extract_android_entries_from_line(file, index, line))
+        end
+
+        entries
+      end
+
+      def extract_android_entries_from_line(file, index, line)
+        @localization_syntax.android_discovery_patterns.flat_map do |resource_type, pattern|
+          line.scan(pattern).filter_map do |captures|
+            key = captures.compact.first
+            next if key.nil? || key.empty?
+
+            DiscoveredLocalization.new(
+              key: key,
+              file: file,
+              line: index + 1,
+              resource_type: resource_type
+            )
+          end
+        end
+      end
+
+      def unescape_source_string(text)
+        AppleStringLiteral.decode(text)
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/searcher.rb

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
-require 'find'
+require 'concurrent'
+require_relative 'path_policy'
+require_relative 'file_classifier'
+require_relative 'localization_syntax'
+require_relative 'searcher/comment_masking'
+require_relative 'searcher/match_filtering'
+require_relative 'searcher/source_discovery'
 
 module I18nContextGenerator
   # Finds where translation keys are used in iOS and Android source code.
   class Searcher
+    include CommentMasking
+    include MatchFiltering
+    include SourceDiscovery
+
     # Represents a code match with surrounding context
     Match = Data.define(:file, :line, :match_line, :context, :enclosing_scope) do
       def initialize(file:, line:, match_line: '', context: '', enclosing_scope: nil)
@@ -12,40 +22,36 @@ module I18nContextGenerator
       end
     end
 
-    # Patterns that indicate false positive matches (not actual localization usage)
-    FALSE_POSITIVE_PATTERNS = [
-      /==\s*["']/,             # String comparisons like == "yes"
-      /["']\s*==/,             # String comparisons like "yes" ==
-      /!=\s*["']/,             # String comparisons like != "no"
-      /["']\s*!=/,             # String comparisons like "no" !=
-      /\.equals\(["']/,        # Java .equals("string")
-      /contentEquals\(["']/    # Kotlin contentEquals
-    ].freeze
-
-    # File extensions to search by platform
-    FILE_EXTENSIONS = {
-      ios: %w[.swift .m .mm .h].freeze,
-      android: %w[.kt .java .xml].freeze,
-      unknown: %w[.swift .m .mm .h .kt .java .xml].freeze
-    }.freeze
-
-    IOS_WRAPPER_DEFINITION_PATTERN =
-      /\b(static\s+)?(?:let|var)\s+(\w+)\s*=\s*(?:NSLocalizedString|String\s*\(\s*localized:|LocalizedStringKey\s*\(|Text\s*\()/
+    # Represents a localization entry discovered directly from source code.
+    DiscoveredLocalization = Data.define(
+      :key, :file, :line, :text, :comment, :resource_type, :locations, :location_groups
+    ) do
+      def initialize(key:, file:, line:, text: nil, comment: nil, resource_type: :string, locations: nil,
+                     location_groups: nil)
+        locations ||= ["#{file}:#{line}"]
+        location_groups ||= [locations]
+        super
+      end
+    end
+
     IOS_TYPE_DECLARATION_PATTERN = /\b(class|struct|enum|extension)\s+(\w+)/
 
-    def initialize(source_paths:, ignore_patterns:, context_lines: 15, platform: nil)
+    def initialize(source_paths:, ignore_patterns:, context_lines: 15, platform: nil, swift_functions: nil)
       @source_paths = source_paths
-      @ignore_patterns = compile_ignore_patterns(ignore_patterns)
+      @path_policy = PathPolicy.new(ignore_patterns: ignore_patterns, roots: source_paths)
       @context_lines = context_lines
       @platform = platform || detect_platform
+      @localization_syntax = LocalizationSyntax.new(swift_functions: swift_functions)
 
       # Cache discovered files for repeated searches
       @files_cache = nil
-      @file_lines_cache = {}
+      @files_cache_mutex = Mutex.new
+      @file_lines_cache = Concurrent::Map.new
+      @searchable_file_lines_cache = Concurrent::Map.new
     end
 
-    def search(key)
-      patterns = build_search_patterns(key)
+    def search(key, resource_type: nil)
+      patterns = build_search_patterns(key, resource_type: resource_type)
       files = discover_files
       direct_matches = []
 
@@ -67,87 +73,47 @@ module I18nContextGenerator
 
     def detect_platform
       @source_paths.each do |path|
-        next unless File.exist?(path)
-
-        if File.directory?(path)
-          Find.find(path) do |f|
-            if File.directory?(f) && ignored?(f)
-              Find.prune
-              next
-            end
-
-            next unless File.file?(f)
-            next if ignored?(f)
-
-            return :ios if f.end_with?('.swift', '.m', '.mm', '.h')
-            return :android if f.end_with?('.kt', '.java')
-          end
-        elsif !ignored?(path) && path.end_with?('.swift', '.m', '.mm', '.h')
-          return :ios
-        elsif !ignored?(path) && path.end_with?('.kt', '.java')
-          return :android
+        @path_policy.each_file(path) do |file|
+          platform = FileClassifier.source_platform(file)
+          return platform if platform
         end
       end
       :unknown
     end
 
-    def compile_ignore_patterns(patterns)
-      patterns.map { |p| glob_to_regex(p) }
-    end
-
-    def glob_to_regex(glob_pattern)
-      # Convert glob pattern to regex
-      # Handle common glob patterns: *, **, ?
-      regex_str = Regexp.escape(glob_pattern)
-                        .gsub('\*\*/', '(.*/)?')  # **/ matches any path (including empty)
-                        .gsub('\*\*', '.*')       # ** matches anything
-                        .gsub('\*', '[^/]*')      # * matches within path segment
-                        .gsub('\?', '.')          # ? matches single char
-      Regexp.new("(?:^|/)#{regex_str}(?:$|/)")
-    end
-
     def discover_files
       return @files_cache if @files_cache
 
-      extensions = FILE_EXTENSIONS[@platform] || FILE_EXTENSIONS[:unknown]
-      files = []
+      @files_cache_mutex.synchronize do
+        return @files_cache if @files_cache
 
-      @source_paths.each do |path|
-        if File.file?(path)
-          files << path if extensions.any? { |ext| path.end_with?(ext) }
-        elsif File.directory?(path)
-          # Build a single glob pattern for all extensions
-          ext_pattern = extensions.size == 1 ? "*#{extensions.first}" : "*{#{extensions.join(',')}}"
-          files.concat(Dir.glob(File.join(path, '**', ext_pattern)))
+        @files_cache = @path_policy.files(@source_paths) do |file|
+          FileClassifier.searchable_source?(file, platform: @platform)
         end
       end
-
-      # Apply ignore patterns and cache
-      @files_cache = files.reject { |f| ignored?(f) }
     end
 
-    def ignored?(file)
-      @ignore_patterns.any? { |pattern| pattern.match?(file) }
+    def ignored?(file, directory: false)
+      @path_policy.ignored?(file, directory: directory)
     end
 
     def search_file(file, patterns, key, enable_multiline: true)
       matches = []
-      lines = []
+      lines = cached_file_lines(file)
+      searchable_lines = searchable_file_lines(file)
       match_indices = Set.new
 
-      # Read file and find all matching line indices in a single pass
-      File.foreach(file).with_index do |line, index|
-        line = line.chomp
-        lines << line
-
+      # Find matching line indices using the comment-masked source while
+      # retaining the original lines for prompt context and output.
+      searchable_lines.each_with_index do |line, index|
         # Check if any pattern matches this line
         match_indices << index if patterns.any? { |pattern| pattern.match?(line) }
       end
 
       # For iOS files, also check for multi-line NSLocalizedString patterns
       # where the function call and key are on different lines
-      if enable_multiline && @platform == :ios && file.end_with?('.swift', '.m', '.mm', '.h')
-        multiline_matches = find_multiline_ios_matches(lines, patterns, key)
+      if enable_multiline && @platform == :ios && FileClassifier.searchable_platform(file) == :ios
+        multiline_matches = find_multiline_ios_matches(searchable_lines, patterns, key)
         match_indices.merge(multiline_matches)
       end
 
@@ -215,29 +181,35 @@ module I18nContextGenerator
       return unless definition_index
 
       definition_line = lines[definition_index]
-      definition_match = IOS_WRAPPER_DEFINITION_PATTERN.match(definition_line)
-      return unless definition_match&.captures&.first
+      definition_match = @localization_syntax.ios_wrapper_definition_pattern.match(definition_line)
+      return unless definition_match
 
       type_path = find_ios_type_path(lines, definition_index)
       return if type_path.empty?
 
       {
         type_path: type_path,
-        member_name: definition_match[2],
+        member_name: definition_match[:member_name],
         definition_file: match.file,
         definition_line: definition_index + 1
       }
     end
 
     def cached_file_lines(file)
-      @file_lines_cache[file] ||= File.readlines(file, chomp: true)
+      @file_lines_cache.compute_if_absent(file) { File.readlines(file, chomp: true) }
+    end
+
+    def searchable_file_lines(file)
+      @searchable_file_lines_cache.compute_if_absent(file) do
+        mask_comments(cached_file_lines(file), file)
+      end
     end
 
     def find_ios_wrapper_definition_index(lines, match_index, lookback: 5)
       start_idx = [0, match_index - lookback].max
 
       match_index.downto(start_idx) do |index|
-        return index if IOS_WRAPPER_DEFINITION_PATTERN.match?(lines[index])
+        return index if @localization_syntax.ios_wrapper_definition_pattern.match?(lines[index])
       end
 
       nil
@@ -282,22 +254,24 @@ module I18nContextGenerator
     # Find matches where localization calls span multiple lines
     # e.g., NSLocalizedString(\n    "key",\n    comment: "...")
     def find_multiline_ios_matches(lines, patterns, key)
-      key_pattern = /["']#{Regexp.escape(key)}["']/
+      key_pattern = /@?"#{LocalizationSyntax.swift_string_content_pattern(key)}"/
+      call_patterns = @localization_syntax.ios_multiline_search_patterns(key)
 
       lines.each_with_index.filter_map do |line, index|
         next if patterns.any? { |p| p.match?(line) }  # Already a single-line match
         next unless key_pattern.match?(line)          # Doesn't contain the key
 
-        index if preceded_by_localization_opener?(lines, index)
+        index if preceded_by_localization_call?(lines, index, call_patterns)
       end.to_set
     end
 
-    def preceded_by_localization_opener?(lines, index, lookback: 5)
+    def preceded_by_localization_call?(lines, index, patterns, lookback: 5)
       start_idx = [0, index - lookback].max
 
       (start_idx...index).reverse_each do |i|
         line = lines[i]
-        return true if IOS_FUNCTION_OPENERS.any? { |opener| opener.match?(line) }
+        snippet = lines[i..index].join("\n")
+        return true if patterns.any? { |pattern| pattern.match?(snippet) }
         return false if line =~ /;\s*$/ || line =~ /\)\s*$/ # Hit a statement boundary
       end
 
@@ -326,122 +300,16 @@ module I18nContextGenerator
       context_parts.join("\n")
     end
 
-    def build_search_patterns(key)
-      pattern_strings = case @platform
-                        when :ios
-                          build_ios_patterns(key)
-                        when :android
-                          build_android_patterns(key)
-                        else
-                          build_ios_patterns(key) + build_android_patterns(key) + [Regexp.escape(key)]
-                        end
-
-      # Pre-compile all patterns for this search
-      pattern_strings.map { |p| Regexp.new(p) }
-    end
-
-    # Extract the base resource name from composite Android keys
-    # e.g., "post_likes_count:one" -> "post_likes_count"
-    #        "days_of_week[0]"     -> "days_of_week"
-    def android_base_key(key)
-      key.sub(/:[a-z]+$/, '').sub(/\[\d+\]$/, '')
+    def build_search_patterns(key, resource_type: nil)
+      @localization_syntax.search_patterns(key, platform: @platform, resource_type: resource_type)
     end
 
-    # Patterns that indicate the start of a localization function call
-    # Used for multi-line matching when the key is on a different line
-    IOS_FUNCTION_OPENERS = [
-      /NSLocalizedString\s*\(\s*$/,
-      /String\s*\(\s*localized:\s*$/,
-      /LocalizedStringKey\s*\(\s*$/,
-      /Text\s*\(\s*$/
-    ].freeze
-    private_constant :IOS_FUNCTION_OPENERS
-
     def build_ios_patterns(key)
-      escaped = Regexp.escape(key)
-      [
-        # NSLocalizedString("key", ...) - most common (Swift and Obj-C)
-        # Note: @? handles optional @ prefix for Objective-C @"string" syntax
-        "NSLocalizedString\\s*\\(\\s*@?[\"']#{escaped}[\"']",
-        # String(localized: "key", ...) - modern Swift
-        "String\\s*\\(\\s*localized:\\s*[\"']#{escaped}[\"']",
-        # LocalizedStringKey("key") - SwiftUI
-        "LocalizedStringKey\\s*\\(\\s*[\"']#{escaped}[\"']",
-        # Text("key") - SwiftUI (when using localized strings)
-        "Text\\s*\\(\\s*[\"']#{escaped}[\"']",
-        # .localized extension pattern
-        "[\"']#{escaped}[\"']\\.localized"
-      ]
+      @localization_syntax.ios_search_patterns(key)
     end
 
-    def build_android_patterns(key)
-      base = android_base_key(key)
-      escaped_base = Regexp.escape(base)
-
-      if key =~ /:[a-z]+$/
-        # Plural key (e.g., "post_likes_count:one") — search by base name in plural resources
-        [
-          "R\\.plurals\\.#{escaped_base}\\b",
-          "@plurals/#{escaped_base}\\b",
-          "getQuantityString\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
-          "\\.getQuantityString\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
-          "pluralStringResource\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
-          "[\\(\\s,=]plurals\\.#{escaped_base}\\b"
-        ]
-      elsif key =~ /\[\d+\]$/
-        # Array key (e.g., "days_of_week[0]") — search by base name in array resources
-        [
-          "R\\.array\\.#{escaped_base}\\b",
-          "@array/#{escaped_base}\\b",
-          "getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
-          "\\.getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
-          "resources\\.getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
-          "[\\(\\s,=]array\\.#{escaped_base}\\b"
-        ]
-      else
-        # Standard string key
-        escaped = Regexp.escape(key)
-        [
-          "R\\.string\\.#{escaped}\\b",
-          "@string/#{escaped}\\b",
-          "getString\\s*\\(\\s*R\\.string\\.#{escaped}",
-          "\\.getString\\s*\\(\\s*R\\.string\\.#{escaped}",
-          "stringResource\\s*\\(\\s*R\\.string\\.#{escaped}",
-          "[\\(\\s,=]string\\.#{escaped}\\b",
-          "getString\\s*\\(\\s*string\\.#{escaped}",
-          "stringResource\\s*\\(\\s*string\\.#{escaped}"
-        ]
-      end
-    end
-
-    def filter_matches(matches, key)
-      seen = Set.new
-      matches.select do |match|
-        location = "#{match.file}:#{match.line}"
-        next false if seen.include?(location)
-        next false if false_positive?(match.match_line, key)
-        next false if translation_file?(match.file)
-
-        seen.add(location)
-      end
-    end
-
-    def false_positive?(line, _key)
-      return false if line.nil? || line.empty?
-
-      FALSE_POSITIVE_PATTERNS.any? { |pattern| pattern.match?(line) }
-    end
-
-    def translation_file?(file)
-      basename = File.basename(file).downcase
-      ext = File.extname(file).downcase
-
-      # Skip translation files - we want code usage, not definitions
-      return true if ext == '.strings'
-      return true if basename == 'strings.xml'
-      return true if file.include?('/res/values') && ext == '.xml'
-
-      false
+    def build_android_patterns(key, resource_type: nil)
+      @localization_syntax.android_search_patterns(key, resource_type: resource_type)
     end
   end
 end