i18n-context-generator 0.4.0 → 0.5.1

data/lib/i18n_context_generator/parsers/xcstrings_parser.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  module Parsers
+    # Parser for Apple string catalogs. Context generation is keyed by the
+    # catalog entry rather than by individual target-language localizations.
+    class XcstringsParser < Base
+      def parse(path)
+        catalog = load_catalog(path)
+        source_language = catalog['sourceLanguage']
+
+        catalog.fetch('strings').filter_map do |key, entry|
+          next if key.empty?
+          next if entry['shouldTranslate'] == false
+
+          TranslationEntry.new(
+            key: key,
+            text: source_text(key, entry, source_language),
+            source_file: path,
+            metadata: {
+              comment: entry['comment'],
+              source_language: source_language,
+              resource_type: :string
+            }.compact
+          )
+        end
+      rescue KeyError => e
+        raise Error, "Failed to parse Apple string catalog #{path}: missing #{e.key.inspect}"
+      rescue Oj::ParseError, TypeError => e
+        raise Error, "Failed to parse Apple string catalog #{path}: #{e.message}"
+      end
+
+      def self.validate_catalog!(catalog, path:)
+        raise TypeError, 'root must be a mapping' unless catalog.is_a?(Hash)
+        raise TypeError, 'sourceLanguage must be a non-empty string' unless nonempty_string?(catalog['sourceLanguage'])
+        raise TypeError, 'strings must be a mapping' unless catalog['strings'].is_a?(Hash)
+
+        valid_entries = catalog['strings'].all? { |key, entry| key.is_a?(String) && entry.is_a?(Hash) }
+        raise TypeError, 'strings must map string keys to entries' unless valid_entries
+
+        catalog
+      rescue TypeError => e
+        raise Error, "Failed to parse Apple string catalog #{path}: #{e.message}"
+      end
+
+      def self.nonempty_string?(value)
+        value.is_a?(String) && !value.empty?
+      end
+      private_class_method :nonempty_string?
+
+      private
+
+      def load_catalog(path)
+        catalog = Oj.load_file(path, mode: :strict)
+        self.class.validate_catalog!(catalog, path: path)
+      end
+
+      def source_text(key, entry, source_language)
+        localization = entry.dig('localizations', source_language)
+        values = string_unit_values(localization)
+        values.empty? ? key : values.join(' | ')
+      end
+
+      def string_unit_values(value)
+        case value
+        when Hash
+          direct_value = value.dig('stringUnit', 'value')
+          return [direct_value] if direct_value.is_a?(String)
+
+          value.values.flat_map { |child| string_unit_values(child) }
+        when Array
+          value.flat_map { |child| string_unit_values(child) }
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/parsers/yaml_parser.rb

@@ -4,16 +4,16 @@ module I18nContextGenerator
   module Parsers
     # Parses YAML translation files (including Rails i18n style) into TranslationEntry objects.
     class YamlParser < Base
+      def initialize(locale: nil)
+        super()
+        @locale = locale
+      end
+
       def parse(path)
         data = YAML.safe_load_file(path, permitted_classes: [])
+        raise Error, "Invalid YAML translation file #{path}: root must be a mapping" unless data.is_a?(Hash)
 
-        # Skip top-level locale key if present (Rails i18n style)
-        # e.g., { "en" => { "hello" => "Hello" } } -> { "hello" => "Hello" }
-        if data.is_a?(Hash) && data.keys.size == 1 && data.values.first.is_a?(Hash)
-          locale_key = data.keys.first
-          # Only skip if it looks like a locale code (2-5 chars)
-          data = data.values.first if locale_key.match?(/\A[a-z]{2}(-[A-Z]{2})?\z/i)
-        end
+        data = locale_root(data, path) if @locale
 
         flatten_keys(data).filter_map do |key, text|
           next if text.nil? || text.to_s.strip.empty?
@@ -24,6 +24,19 @@ module I18nContextGenerator
             source_file: path
           )
         end
+      rescue Psych::SyntaxError => e
+        raise Error, "Failed to parse YAML translation file #{path}: #{e.problem} at line #{e.line}, column #{e.column}"
+      rescue Psych::Exception => e
+        raise Error, "Failed to parse YAML translation file #{path}: #{e.message}"
+      end
+
+      private
+
+      def locale_root(data, path)
+        root = data[@locale]
+        raise Error, "YAML translation file #{path} does not contain locale root #{@locale.inspect}" unless root.is_a?(Hash)
+
+        root
       end
     end
   end

data/lib/i18n_context_generator/path_policy.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'find'
+require 'pathname'
+
+module I18nContextGenerator
+  # Compiles client ignore globs once and applies them consistently while
+  # traversing configured source roots.
+  class PathPolicy
+    def initialize(ignore_patterns:, roots: [])
+      @ignore_patterns = self.class.compile_globs(ignore_patterns)
+      @roots = roots.map { |root| normalize(root) }.uniq
+    end
+
+    def ignored?(path, directory: false)
+      candidates = path_candidates(path, directory: directory)
+      @ignore_patterns.any? do |pattern|
+        candidates.any? { |candidate| pattern.match?(candidate) }
+      end
+    end
+
+    def each_file(path)
+      return enum_for(__method__, path) unless block_given?
+
+      if File.file?(path)
+        yield path unless ignored?(path)
+        return
+      end
+      return unless File.directory?(path)
+      return if ignored?(path, directory: true)
+
+      Find.find(path) do |candidate|
+        if File.directory?(candidate)
+          Find.prune if candidate != path && ignored?(candidate, directory: true)
+          next
+        end
+
+        yield candidate unless ignored?(candidate)
+      end
+    end
+
+    def files(paths, &predicate)
+      seen = {}
+      paths.each_with_object([]) do |path, files|
+        each_file(path) do |file|
+          next if predicate && !predicate.call(file)
+
+          identity = File.expand_path(file)
+          next if seen[identity]
+
+          seen[identity] = true
+          files << file
+        end
+      end
+    end
+
+    class << self
+      def compile_globs(patterns)
+        Array(patterns).map { |pattern| glob_to_regex(pattern) }.freeze
+      end
+
+      def glob_to_regex(glob_pattern)
+        regex = Regexp.escape(normalize(glob_pattern))
+                      .gsub('\*\*/', '(.*/)?')
+                      .gsub('\*\*', '.*')
+                      .gsub('\*', '[^/]*')
+                      .gsub('\?', '.')
+        Regexp.new("(?:^|/)#{regex}(?:$|/)")
+      end
+
+      private
+
+      def normalize(path)
+        path.to_s.tr('\\', '/')
+      end
+    end
+
+    private
+
+    def path_candidates(path, directory:)
+      normalized_path = normalize(path)
+      candidates = [normalized_path]
+
+      @roots.each do |root|
+        relative = relative_to_root(normalized_path, root)
+        candidates << relative if relative && !relative.empty?
+      end
+
+      return candidates.uniq unless directory
+
+      candidates.flat_map do |candidate|
+        [candidate, candidate.end_with?('/') ? candidate : "#{candidate}/"]
+      end.uniq
+    end
+
+    def relative_to_root(path, root)
+      return '' if path == root
+
+      prefix = root.end_with?('/') ? root : "#{root}/"
+      path.delete_prefix(prefix) if path.start_with?(prefix)
+    end
+
+    def normalize(path)
+      self.class.send(:normalize, Pathname.new(path.to_s).cleanpath.to_s)
+    end
+  end
+end

data/lib/i18n_context_generator/platform_validator.rb

@@ -1,55 +1,45 @@
 # frozen_string_literal: true
 
-require 'find'
+require_relative 'path_policy'
+require_relative 'file_classifier'
 
 module I18nContextGenerator
   # Validates that a run targets a single mobile platform after applying ignore rules.
   class PlatformValidator
     def initialize(config)
       @config = config
-      @ignore_patterns = @config.ignore_patterns.map { |pattern| glob_to_regex(pattern) }
+      @path_policy = PathPolicy.new(ignore_patterns: @config.ignore_patterns, roots: @config.source_paths)
     end
 
     def validate!
       platforms = (@config.translations.flat_map { |path| translation_platforms_for_path(path) } +
         @config.source_paths.flat_map { |path| source_platforms_for_path(path) }).uniq
 
-      return if platforms.size <= 1
+      raise Error, 'Mixed iOS and Android runs are not supported. Split them into separate invocations or config files.' if platforms.size > 1
 
-      raise Error, 'Mixed iOS and Android runs are not supported. Split them into separate invocations or config files.'
+      detected_platform = platforms.first
+      requested_platform = @config.platform&.to_sym
+      raise Error, "Configured platform #{requested_platform} conflicts with detected #{detected_platform} inputs" if requested_platform && detected_platform && requested_platform != detected_platform
+
+      resolved_platform = requested_platform || detected_platform || :unknown
+      validate_code_write_back!(resolved_platform)
+
+      resolved_platform
     end
 
     private
 
     def translation_platforms_for_path(path)
-      basename = File.basename(path).downcase
-      ext = File.extname(path).downcase
-
-      case ext
-      when '.strings'
-        [:ios]
-      when '.xml'
-        basename == 'strings.xml' || path.include?('/res/values') ? [:android] : []
-      else
-        []
-      end
+      platform = FileClassifier.translation_platform(path)
+      platform ? [platform] : []
     end
 
     def source_platforms_for_path(path)
       return [] unless File.exist?(path)
 
-      if File.file?(path)
-        platform = source_platform_for_file(path)
-        return platform ? [platform] : []
-      end
-
       platforms = []
-
-      Find.find(path) do |file|
-        next unless File.file?(file)
-        next if ignored_source_file?(file)
-
-        platform = source_platform_for_file(file)
+      @path_policy.each_file(path) do |file|
+        platform = FileClassifier.source_platform(file)
         next unless platform
         next if platforms.include?(platform)
 
@@ -60,33 +50,17 @@ module I18nContextGenerator
       platforms
     end
 
-    def source_platform_for_file(path)
-      case File.extname(path).downcase
-      when '.swift', '.m', '.mm', '.h'
-        :ios
-      when '.kt', '.java'
-        :android
-      end
-    end
+    def validate_code_write_back!(platform)
+      return unless @config.write_back_to_code
 
-    def ignored_source_file?(path)
-      @ignore_patterns.any? do |pattern|
-        candidates = [path]
-        @config.source_paths.each do |root|
-          prefix = root.end_with?('/') ? root : "#{root}/"
-          candidates << path.delete_prefix(prefix) if path.start_with?(prefix)
-        end
-        candidates.any? { |candidate| pattern.match?(candidate) }
-      end
+      raise Error, 'write_back_to_code is supported only for iOS Swift sources' unless platform == :ios
+      return if @config.source_paths.any? { |path| contains_writable_swift_source?(path) }
+
+      raise Error, 'write_back_to_code requires at least one non-ignored Swift source file'
     end
 
-    def glob_to_regex(glob_pattern)
-      regex_str = Regexp.escape(glob_pattern)
-                        .gsub('\*\*/', '(.*/)?')
-                        .gsub('\*\*', '.*')
-                        .gsub('\*', '[^/]*')
-                        .gsub('\?', '.')
-      Regexp.new("(?:^|/)#{regex_str}(?:$|/)")
+    def contains_writable_swift_source?(path)
+      @path_policy.each_file(path).any? { |file| FileClassifier.swift_source?(file) }
     end
   end
 end

data/lib/i18n_context_generator/run_metrics.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  RUN_METRICS_PRICING_AS_OF = '2026-07-23'
+  RUN_METRICS_PRICES_PER_MILLION = {
+    ['anthropic', 'claude-sonnet-4-6'] => { input: 3.0, output: 15.0 },
+    ['openai', 'gpt-5-mini'] => { input: 0.25, output: 2.0 }
+  }.freeze
+
+  # Aggregates run-local provider and cache telemetry. Cost is an estimate based
+  # on documented standard list prices for the built-in default models.
+  RunMetrics = Data.define(
+    :request_count, :cache_hits, :input_tokens, :output_tokens,
+    :retry_count, :estimated_cost_usd, :cost_model, :cost_pricing_as_of
+  ) do
+    def self.from(results, provider:, model:)
+      input_tokens = results.sum { |result| result.input_tokens.to_i }
+      output_tokens = results.sum { |result| result.output_tokens.to_i }
+      price = RUN_METRICS_PRICES_PER_MILLION[[provider.to_s, model.to_s]]
+      estimated_cost = ((input_tokens * price[:input]) + (output_tokens * price[:output])) / 1_000_000.0 if price
+
+      new(
+        request_count: results.sum { |result| result.request_count.to_i },
+        cache_hits: results.count(&:cache_hit),
+        input_tokens: input_tokens,
+        output_tokens: output_tokens,
+        retry_count: results.sum { |result| result.retries.to_i },
+        estimated_cost_usd: estimated_cost&.round(8),
+        cost_model: price ? model.to_s : nil,
+        cost_pricing_as_of: price ? RUN_METRICS_PRICING_AS_OF : nil
+      )
+    end
+
+    def to_h
+      {
+        request_count: request_count,
+        cache_hits: cache_hits,
+        input_tokens: input_tokens,
+        output_tokens: output_tokens,
+        retry_count: retry_count,
+        estimated_cost_usd: estimated_cost_usd,
+        cost_model: cost_model,
+        cost_pricing_as_of: cost_pricing_as_of
+      }
+    end
+  end
+end

data/lib/i18n_context_generator/searcher/comment_masking.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  class Searcher
+    # Masks comments while preserving line numbers and character positions.
+    module CommentMasking
+      private
+
+      def mask_comments(lines, file)
+        File.extname(file).downcase == '.xml' ? mask_xml_comments(lines) : mask_c_style_comments(lines)
+      end
+
+      def mask_c_style_comments(lines)
+        block_depth = 0
+
+        lines.map do |line|
+          masked = line.dup
+          quote = nil
+          escaped = false
+          index = 0
+
+          while index < line.length
+            pair = line[index, 2]
+
+            if block_depth.positive?
+              if pair == '/*'
+                mask_characters!(masked, index, 2)
+                block_depth += 1
+                index += 2
+              elsif pair == '*/'
+                mask_characters!(masked, index, 2)
+                block_depth -= 1
+                index += 2
+              else
+                mask_characters!(masked, index, 1)
+                index += 1
+              end
+              next
+            end
+
+            if quote
+              if escaped
+                escaped = false
+              elsif line[index] == '\\'
+                escaped = true
+              elsif line[index] == quote
+                quote = nil
+              end
+              index += 1
+              next
+            end
+
+            if pair == '//'
+              mask_characters!(masked, index, line.length - index)
+              break
+            elsif pair == '/*'
+              mask_characters!(masked, index, 2)
+              block_depth = 1
+              index += 2
+            elsif ['"', "'"].include?(line[index])
+              quote = line[index]
+              index += 1
+            else
+              index += 1
+            end
+          end
+
+          masked
+        end
+      end
+
+      def mask_xml_comments(lines)
+        in_comment = false
+
+        lines.map do |line|
+          masked = line.dup
+          index = 0
+
+          while index < line.length
+            if in_comment
+              closing_index = line.index('-->', index)
+              if closing_index
+                mask_characters!(masked, index, closing_index + 3 - index)
+                index = closing_index + 3
+                in_comment = false
+              else
+                mask_characters!(masked, index, line.length - index)
+                break
+              end
+            else
+              opening_index = line.index('<!--', index)
+              break unless opening_index
+
+              closing_index = line.index('-->', opening_index + 4)
+              if closing_index
+                mask_characters!(masked, opening_index, closing_index + 3 - opening_index)
+                index = closing_index + 3
+              else
+                mask_characters!(masked, opening_index, line.length - opening_index)
+                in_comment = true
+                break
+              end
+            end
+          end
+
+          masked
+        end
+      end
+
+      def mask_characters!(text, start, length)
+        text[start, length] = ' ' * length
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/searcher/match_filtering.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  class Searcher
+    # Removes duplicate, translation-definition, and non-localization matches.
+    module MatchFiltering
+      private
+
+      def filter_matches(matches, key)
+        seen = Set.new
+        explicit_patterns = explicit_localization_patterns(key)
+        matches.select do |match|
+          location = "#{match.file}:#{match.line}"
+          next false if seen.include?(location)
+          next false if false_positive?(match.match_line, key, explicit_patterns)
+          next false if translation_file?(match.file)
+
+          seen.add(location)
+        end
+      end
+
+      def false_positive?(line, key, explicit_patterns)
+        return false if line.nil? || line.empty?
+        return false if explicit_localization_usage?(line, explicit_patterns)
+
+        quoted_key = "[\"']#{Regexp.escape(key)}[\"']"
+        comparison_patterns = [
+          /==\s*#{quoted_key}/,
+          /#{quoted_key}\s*==/,
+          /!=\s*#{quoted_key}/,
+          /#{quoted_key}\s*!=/,
+          /\.equals\(\s*#{quoted_key}/,
+          /contentEquals\(\s*#{quoted_key}/
+        ]
+
+        comparison_patterns.any? { |pattern| pattern.match?(line) }
+      end
+
+      def explicit_localization_patterns(key)
+        (build_ios_patterns(key) + build_android_patterns(key)).map { |pattern| Regexp.new(pattern) }
+      end
+
+      def explicit_localization_usage?(line, patterns)
+        patterns.any? { |pattern| pattern.match?(line) }
+      end
+
+      def translation_file?(file)
+        !FileClassifier.translation_platform(file).nil?
+      end
+    end
+  end
+end