i18n-context-generator 0.4.0 → 0.5.0

data/lib/i18n_context_generator/context_extractor.rb

@@ -2,6 +2,11 @@
 
 require_relative 'context_extractor/source_filters'
 require_relative 'context_extractor/run_logging'
+require_relative 'context_extractor/source_entries'
+require_relative 'context_extractor/translation_filters'
+require_relative 'context_extractor/cache_identity'
+require_relative 'context_extractor/extraction_result'
+require_relative 'context_extractor/workflow'
 
 module I18nContextGenerator
   # Main orchestrator that parses translation files, searches source code for usages,
@@ -10,45 +15,36 @@ module I18nContextGenerator
     include Writers::Helpers
     include SourceFilters
     include RunLogging
+    include SourceEntries
+    include TranslationFilters
+    include CacheIdentity
+    include Workflow
 
-    # Result for a single translation key
-    ExtractionResult = Data.define(:key, :text, :description, :source_file, :ui_element, :tone,
-                                   :max_length, :locations, :error) do
-      def initialize(key:, text:, description:, source_file: nil, ui_element: nil, tone: nil,
-                     max_length: nil, locations: [], error: nil)
-        super
-      end
-
-      def to_h
-        {
-          key: key,
-          text: text,
-          description: description,
-          source_file: source_file,
-          ui_element: ui_element,
-          tone: tone,
-          max_length: max_length,
-          locations: locations,
-          error: error
-        }
-      end
-    end
-
-    attr_reader :results, :errors
+    attr_reader :results, :errors, :metrics
 
-    def initialize(config)
+    def initialize(config, log_output: nil, structured_output: nil, patch_output: nil,
+                   quiet: false, progress: true)
       @config = config
+      @configured_log_output = log_output
+      @structured_output = structured_output
+      @patch_output = patch_output
+      @quiet = quiet
+      @progress_enabled = progress && !quiet
       @results = Concurrent::Array.new
       @errors = Concurrent::Array.new
+      @metrics = RunMetrics.from([], provider: @config.provider, model: resolved_model)
 
       # Defer initialization of expensive resources
       @searcher = nil
       @llm = nil
       @cache = nil
+      @supplemental_context = nil
+      @cache_context_sources = nil
     end
 
     def run
-      PlatformValidator.new(@config).validate!
+      @config.validate!
+      @platform = PlatformValidator.new(@config).validate!
 
       entries = load_entries
       entries = filter_entries(entries) if @config.key_filter
@@ -62,25 +58,27 @@ module I18nContextGenerator
 
       log_loaded_entries(entries.size)
 
-      if @config.dry_run
-        puts "\nDry run - would process these keys:"
-        entries.first(20).each { |e| puts "  - #{e.key}: #{truncate(e.text, 50)}" }
-        puts "  ... and #{entries.size - 20} more" if entries.size > 20
-        return
+      return if pre_extraction_stage_handled?(entries)
+
+      # Resolve shared prompt inputs and provider state on the caller thread so
+      # file reads, digest work, and configuration errors are not duplicated by workers.
+      context_sources = supplemental_context
+      cache_context_sources
+      llm_client = llm
+      unless context_sources.empty?
+        llm_client.validate_supplemental_context!(
+          supplemental_context: context_sources,
+          redact_prompts: @config.redact_prompts,
+          max_prompt_chars: @config.max_prompt_chars
+        )
       end
-
       process_entries(entries)
+      @metrics = RunMetrics.from(@results, provider: @config.provider, model: resolved_model)
 
-      if @config.output_path
-        write_output
-        puts "\nWrote #{@results.size} results to #{@config.output_path}"
-      end
-
-      write_back_to_source if @config.write_back
-
-      write_back_to_code if @config.write_back_to_code
+      deliver_results
 
-      puts "Errors: #{@errors.size}" if @errors.any?
+      log "Errors: #{@errors.size}" if @errors.any?
+      log_metrics
     end
 
     private
@@ -89,95 +87,49 @@ module I18nContextGenerator
       @searcher ||= Searcher.new(
         source_paths: @config.source_paths,
         ignore_patterns: @config.ignore_patterns,
-        context_lines: @config.context_lines
+        context_lines: @config.context_lines,
+        platform: @platform,
+        swift_functions: @config.swift_functions
       )
     end
 
     def llm
-      @llm ||= LLM::Client.for(@config.provider)
+      @llm ||= LLM::Client.for(@config.provider, endpoint: @config.endpoint)
     end
 
     def cache
-      @cache ||= Cache.new(enabled: !@config.no_cache)
+      @cache ||= Cache.new(enabled: !@config.no_cache, directory: @config.cache_dir)
+    end
+
+    def supplemental_context
+      @supplemental_context ||= SupplementalContext.load(
+        files: @config.context_files,
+        runtime: @config.supplemental_context
+      )
     end
 
     def load_translations
       return @load_translations if defined?(@load_translations)
 
-      @load_translations = @config.translations.uniq.flat_map do |path|
-        unless File.exist?(path)
-          warn "Translation file not found: #{path}"
-          next []
-        end
-
-        parser = Parsers::Base.for(path)
+      entries = @config.translations.flat_map do |path|
+        parser = Parsers::Base.for(path, locale: @config.translation_locales[path])
         parser.parse(path)
       end
-    end
 
-    def load_entries
-      case normalized_discovery_mode
-      when 'source'
-        load_source_entries
-      when 'translations'
-        load_translations
-      else
-        auto_discovery_entries
-      end
-    end
-
-    def auto_discovery_entries
-      return load_source_entries if @config.translations.empty?
-
-      load_translations
-    end
-
-    def load_source_entries
-      translation_lookup = load_translation_lookup
-      discovered_entries = filter_source_entries(searcher.discover_localization_entries)
-
-      discovered_entries.map do |entry|
-        hydrated_entry = translation_lookup[entry.key]
-        translation_comment = hydrated_entry&.metadata&.dig(:comment)
-        source_comment = entry.comment
-        metadata = {}
-        metadata[:comment] = translation_comment || source_comment if translation_comment || source_comment
-        metadata[:source_location] = "#{entry.file}:#{entry.line}"
-
-        Parsers::TranslationEntry.new(
-          key: entry.key,
-          text: hydrated_entry&.text || entry.text || entry.key,
-          source_file: hydrated_entry&.source_file,
-          metadata: metadata
-        )
-      end
-    end
-
-    def load_translation_lookup
-      return @load_translation_lookup if defined?(@load_translation_lookup)
-
-      @load_translation_lookup = load_translations.each_with_object({}) do |entry, lookup|
-        lookup[entry.key] ||= entry
-      end
-    end
-
-    def normalized_discovery_mode
-      @config.discovery_mode.to_s.downcase
-    end
-
-    def translation_backed_discovery?
-      return true if normalized_discovery_mode == 'translations'
-      return false if normalized_discovery_mode == 'source'
-
-      @config.translations.any?
-    end
-
-    def entry_label_for_logging
-      translation_backed_discovery? ? 'translation keys' : 'source localization entries'
+      @load_translations = validate_translation_entries(entries)
+    rescue I18nContextGenerator::Error
+      raise
+    rescue StandardError => e
+      raise Error, "Failed to load translations: #{e.message}"
     end
 
     def filter_entries(entries)
-      patterns = @config.key_filter.split(',').map do |pattern|
+      configured_patterns = if @config.key_filter.is_a?(Array)
+                              @config.key_filter
+                            else
+                              @config.key_filter.split(',')
+                            end
+      patterns = configured_patterns.map do |pattern|
         escaped = Regexp.escape(pattern.strip).gsub('\*', '.*')
         Regexp.new("^#{escaped}$")
       end
@@ -187,109 +139,93 @@ module I18nContextGenerator
       end
     end
 
-    def filter_by_diff(entries)
-      git_diff = GitDiff.new(base_ref: @config.diff_base)
-      changed_keys = git_diff.changed_keys(@config.translations)
-
-      if changed_keys.empty?
-        puts "No changes detected in translation files since #{@config.diff_base}"
-        return []
-      end
-
-      puts "Found #{changed_keys.size} changed keys in git diff"
-
-      entries.select do |entry|
-        changed_keys.include?(entry.key) || changed_keys.include?(android_base_key(entry.key))
-      end
-    end
-
-    # Extract the base resource name from composite Android keys
-    # e.g., "post_likes_count:one" -> "post_likes_count"
-    #        "days_of_week[0]"     -> "days_of_week"
-    def android_base_key(key)
-      key.sub(/:[a-z]+$/, '').sub(/\[\d+\]$/, '')
-    end
-
     def filter_by_range(entries)
       start_idx = 0
       end_idx = entries.size - 1
 
       if @config.start_key
         found_idx = entries.find_index { |e| e.key == @config.start_key }
-        if found_idx
-          start_idx = found_idx
-        else
-          puts "Warning: start_key '#{@config.start_key}' not found, starting from beginning"
-        end
+        raise Error, "start_key not found: #{@config.start_key}" unless found_idx
+
+        start_idx = found_idx
       end
 
       if @config.end_key
-        found_idx = entries.find_index { |e| e.key == @config.end_key }
-        if found_idx
-          end_idx = found_idx
-        else
-          puts "Warning: end_key '#{@config.end_key}' not found, processing to end"
-        end
+        found_idx = entries.rindex { |e| e.key == @config.end_key }
+        raise Error, "end_key not found: #{@config.end_key}" unless found_idx
+
+        end_idx = found_idx
       end
 
+      raise Error, 'start_key must not come after end_key' if start_idx > end_idx
+
       range_info = []
       range_info << "from '#{@config.start_key}'" if @config.start_key
       range_info << "to '#{@config.end_key}'" if @config.end_key
-      puts "Filtering #{range_info.join(' ')}: keys #{start_idx + 1} to #{end_idx + 1}"
+      log "Filtering #{range_info.join(' ')}: keys #{start_idx + 1} to #{end_idx + 1}"
 
       entries[start_idx..end_idx]
     end
 
     def process_entries(entries)
-      # Ensure output is not buffered
-      $stdout.sync = true
+      # Results expose changed source locations even during translation-backed
+      # discovery. Resolve the diff once on the caller thread before workers can
+      # race to initialize the lazy filter.
+      source_line_filter if @config.diff_base
 
-      progress = TTY::ProgressBar.new(
-        '[:bar] :current/:total :percent :eta :key',
-        total: entries.size,
-        width: 30,
-        output: $stdout
-      )
+      progress = build_progress(entries.size)
 
       # Use a thread pool for concurrent processing
       pool = Concurrent::FixedThreadPool.new(@config.concurrency)
-      semaphore = Concurrent::Semaphore.new(@config.concurrency)
-      current_key = Concurrent::AtomicReference.new('')
 
       entries.each do |entry|
         pool.post do
-          semaphore.acquire
-          begin
-            current_key.set(truncate(entry.key, 40))
-            result = process_entry(entry)
-            @results << result
-            @errors << result if result.error
-          rescue StandardError => e
-            # Capture errors as results so they're visible in output
-            result = ExtractionResult.new(
-              key: entry.key,
-              text: entry.text,
-              description: 'Processing failed',
-              source_file: entry.source_file,
-              error: e.message
-            )
-            @results << result
-            @errors << result
-          ensure
-            semaphore.release
-            progress.advance(key: current_key.get)
-          end
+          result = process_entry(entry)
+          @results << result
+          @errors << result if result.error
+        rescue StandardError => e
+          # Capture errors as results so they're visible in output
+          result = ExtractionResult.new(
+            key: entry.key,
+            text: entry.text,
+            description: 'Processing failed',
+            source_file: entry.source_file,
+            translation_key: translation_key_for(entry),
+            changed_translation_locations: changed_translation_locations_for(entry),
+            status: :error,
+            error: e.message
+          )
+          @results << result
+          @errors << result
+        ensure
+          progress&.advance(key: truncate(entry.key, 40))
         end
       end
 
       pool.shutdown
       pool.wait_for_termination
-      puts # New line after progress bar
+      log if progress # New line after progress bar
+    end
+
+    def build_progress(total)
+      return unless @progress_enabled
+
+      TTY::ProgressBar.new(
+        '[:bar] :current/:total :percent :eta :key',
+        total: total,
+        width: 30,
+        output: log_output
+      )
     end
 
     def process_entry(entry)
       # Search for key usage in code first — needed for both cache key and LLM prompt
-      matches = searcher.search(entry.key)
+      resource_type = entry.metadata&.dig(:resource_type)
+      matches = if resource_type
+                  searcher.search(entry.key, resource_type: resource_type)
+                else
+                  searcher.search(entry.key)
+                end
       comment = @config.include_translation_comments ? entry.metadata&.dig(:comment) : nil
 
       if matches.empty?
@@ -298,28 +234,22 @@ module I18nContextGenerator
           text: entry.text,
           description: 'No usage found in source code',
           source_file: entry.source_file,
+          translation_key: translation_key_for(entry),
+          changed_translation_locations: changed_translation_locations_for(entry),
+          status: :no_usage,
           locations: []
         )
       end
 
       # Limit matches to avoid huge prompts
       matches = matches.first(@config.max_matches_per_key)
+      context_sources = supplemental_context
 
-      # Build a cache context digest from all prompt-shaping inputs so the cache
-      # invalidates when source code, comments, or model change
-      cache_ctx = [
-        matches.map { |m| "#{m.file}:#{m.line}:#{m.match_line}:#{m.enclosing_scope}:#{m.context}" }.sort.join("\0"),
-        "comment:#{comment}",
-        "provider:#{@config.provider}",
-        "model:#{@config.model}",
-        "include_file_paths:#{@config.include_file_paths}",
-        "redact_prompts:#{@config.redact_prompts}"
-      ].join("\n")
+      cache_ctx = cache_context(entry, matches, comment)
 
       # Check cache with match context included
-      if (cached = cache.get(entry.key, entry.text, context: cache_ctx))
-        return ExtractionResult.new(source_file: entry.source_file, **cached.transform_keys(&:to_sym))
-      end
+      cached = cache.get(entry.key, entry.text, context: cache_ctx)
+      return cached_extraction_result(entry, cached, matches) if cached && !(cached[:error] || cached['error'])
 
       # Get context from LLM
       llm_result = llm.generate_context(
@@ -329,9 +259,12 @@ module I18nContextGenerator
         model: @config.model,
         comment: comment,
         include_file_paths: @config.include_file_paths,
-        redact_prompts: @config.redact_prompts
+        redact_prompts: @config.redact_prompts,
+        max_prompt_chars: @config.max_prompt_chars,
+        supplemental_context: context_sources
       )
 
+      result_locations = result_locations_for(entry, matches)
       result = ExtractionResult.new(
         key: entry.key,
         text: entry.text,
@@ -340,14 +273,52 @@ module I18nContextGenerator
         ui_element: llm_result.ui_element,
         tone: llm_result.tone,
         max_length: llm_result.max_length,
-        locations: result_locations_for(entry, matches),
+        confidence: llm_result.confidence,
+        ambiguity_reason: llm_result.ambiguity_reason,
+        request_count: llm_result.request_count,
+        input_tokens: llm_result.input_tokens,
+        output_tokens: llm_result.output_tokens,
+        retries: llm_result.retries,
+        locations: result_locations,
+        **changed_location_attributes_for(entry, result_locations),
+        translation_key: translation_key_for(entry),
+        changed_translation_locations: changed_translation_locations_for(entry),
+        status: llm_result.error ? :error : :success,
         error: llm_result.error
       )
 
-      cache.set(entry.key, entry.text, result.to_h.except(:source_file), context: cache_ctx)
+      unless result.error
+        cache.set(
+          entry.key,
+          entry.text,
+          result.to_h.except(
+            :source_file, :changed_locations, :changed_location_groups, :changed_translation_locations,
+            :cache_hit, :request_count, :input_tokens, :output_tokens, :retries
+          ),
+          context: cache_ctx
+        )
+      end
       result
     end
 
+    def cached_extraction_result(entry, cached, matches)
+      attributes = cached.transform_keys(&:to_sym).except(
+        :source_file, :locations, :changed_locations, :changed_location_groups,
+        :translation_key, :changed_translation_locations,
+        :cache_hit, :request_count, :input_tokens, :output_tokens, :retries
+      )
+      locations = result_locations_for(entry, matches)
+      ExtractionResult.new(
+        source_file: entry.source_file,
+        locations: locations,
+        **changed_location_attributes_for(entry, locations),
+        translation_key: translation_key_for(entry),
+        changed_translation_locations: changed_translation_locations_for(entry),
+        cache_hit: true,
+        **attributes
+      )
+    end
+
     def write_output
       writer = case @config.output_format.to_s.downcase
                when 'json'
@@ -356,7 +327,12 @@ module I18nContextGenerator
                  Writers::CsvWriter.new
                end
 
-      writer.write(@results, @config.output_path)
+      writer.write(
+        @results,
+        @config.output_path,
+        metrics: @metrics,
+        output: @structured_output || $stdout
+      )
     end
 
     def write_back_to_source
@@ -369,33 +345,44 @@ module I18nContextGenerator
         relevant_results = @results.select { |result| result_matches_source_path?(result, path) }
         next if relevant_results.empty?
 
-        writer.write(relevant_results, path)
-        puts "Updated #{path} with context comments"
+        updated = writer.write(relevant_results, path)
+        log "Updated #{path} with context comments" if updated
       end
     end
 
     def write_back_to_code
-      swift_writer = Writers::SwiftWriter.new(
-        functions: @config.swift_functions,
-        context_prefix: @config.context_prefix,
-        context_mode: @config.context_mode
-      )
-
       updated_count = 0
       results_by_key = build_results_by_key_for_code_write_back
+      return if results_by_key.empty?
 
-      @config.source_paths.each do |source_path|
-        swift_files = find_swift_files(source_path, ignore_patterns: @config.ignore_patterns)
-
-        swift_files.each do |swift_file|
-          if swift_writer.update_file(swift_file, results_by_key)
-            updated_count += 1
-            puts "Updated #{swift_file} with context comments"
-          end
+      swift_files_for_write_back.each do |swift_file|
+        if swift_writer.update_file(swift_file, results_by_key)
+          updated_count += 1
+          log "Updated #{swift_file} with context comments"
         end
       end
 
-      puts "Updated #{updated_count} Swift files with context comments" if updated_count.positive?
+      log "Updated #{updated_count} Swift files with context comments" if updated_count.positive?
+    end
+
+    def validate_translation_entries(entries)
+      invalid_entry = entries.find do |entry|
+        !entry.key.is_a?(String) || entry.key.empty? ||
+          !entry.text.is_a?(String) ||
+          !entry.source_file.is_a?(String) || entry.source_file.empty?
+      end
+      if invalid_entry
+        source = invalid_entry.source_file || 'unknown source'
+        raise Error, "Invalid translation entry in #{source}: key, text, and source file must be strings"
+      end
+
+      grouped = entries.group_by { |entry| [File.expand_path(entry.source_file), entry.key] }
+      conflicts = grouped.filter_map do |(source_file, key), duplicates|
+        "#{source_file}:#{key}" if duplicates.map(&:text).uniq.size > 1
+      end
+      raise Error, "Conflicting duplicate translation keys: #{conflicts.join(', ')}" if conflicts.any?
+
+      entries.uniq { |entry| [File.expand_path(entry.source_file), entry.key, entry.text] }
     end
 
     def build_results_by_key_for_code_write_back
@@ -416,7 +403,6 @@ module I18nContextGenerator
     end
 
     def source_writer_for(path)
-      basename = File.basename(path).downcase
       ext = File.extname(path).downcase
 
       case ext
@@ -425,8 +411,13 @@ module I18nContextGenerator
           context_prefix: @config.context_prefix,
           context_mode: @config.context_mode
         )
+      when '.xcstrings'
+        Writers::XcstringsWriter.new(
+          context_prefix: @config.context_prefix,
+          context_mode: @config.context_mode
+        )
       when '.xml'
-        if basename == 'strings.xml' || path.include?('/res/values')
+        if FileClassifier.android_translation_file?(path)
           Writers::AndroidXmlWriter.new(
             context_prefix: @config.context_prefix,
             context_mode: @config.context_mode

data/lib/i18n_context_generator/file_classifier.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  # Owns source and translation file-type classification. Header files remain
+  # searchable for iOS runs but are deliberately not platform evidence because
+  # generic C headers also occur in Android projects.
+  module FileClassifier
+    IOS_SOURCE_EXTENSIONS = %w[.swift .m .mm].freeze
+    IOS_SEARCH_EXTENSIONS = (IOS_SOURCE_EXTENSIONS + ['.h']).freeze
+    ANDROID_SOURCE_EXTENSIONS = %w[.kt .java].freeze
+    SEARCH_EXTENSIONS = {
+      ios: IOS_SEARCH_EXTENSIONS,
+      android: (ANDROID_SOURCE_EXTENSIONS + ['.xml']).freeze,
+      unknown: (IOS_SEARCH_EXTENSIONS + ANDROID_SOURCE_EXTENSIONS + ['.xml']).freeze
+    }.freeze
+
+    module_function
+
+    def source_platform(path)
+      extension = File.extname(path).downcase
+      return :ios if IOS_SOURCE_EXTENSIONS.include?(extension)
+      return :android if ANDROID_SOURCE_EXTENSIONS.include?(extension)
+
+      :android if android_source_xml?(path)
+    end
+
+    def searchable_platform(path, platform_hint: nil)
+      return :ios if File.extname(path).downcase == '.h'
+      return :android if File.extname(path).downcase == '.xml' && platform_hint&.to_sym == :android
+
+      source_platform(path)
+    end
+
+    def searchable_source?(path, platform: :unknown)
+      extensions = SEARCH_EXTENSIONS.fetch(platform.to_sym, SEARCH_EXTENSIONS[:unknown])
+      return false unless extensions.include?(File.extname(path).downcase)
+      return platform.to_sym == :android || android_source_xml?(path) if File.extname(path).downcase == '.xml'
+
+      true
+    end
+
+    def translation_platform(path)
+      case File.extname(path).downcase
+      when '.strings', '.xcstrings' then :ios
+      when '.xml' then :android if android_translation_file?(path)
+      end
+    end
+
+    def android_resource_xml?(path)
+      File.extname(path).downcase == '.xml' && path_components(path).include?('res')
+    end
+
+    def android_source_xml?(path)
+      android_resource_xml?(path) || File.basename(path).casecmp('AndroidManifest.xml').zero?
+    end
+
+    def android_translation_file?(path)
+      return false unless File.extname(path).downcase == '.xml'
+      return true if File.basename(path).casecmp('strings.xml').zero?
+
+      path_components(path).each_cons(2).any? { |first, second| first == 'res' && second.start_with?('values') }
+    end
+
+    def swift_source?(path)
+      File.extname(path).downcase == '.swift'
+    end
+
+    def path_components(path)
+      path.to_s.tr('\\', '/').split('/').reject(&:empty?)
+    end
+  end
+end