i18n-context-generator 0.4.0 → 0.5.1

data/lib/i18n_context_generator/llm/openai_compatible.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  module LLM
+    # Explicit OpenAI Responses-compatible endpoint support. A separate
+    # credential variable prevents local or third-party endpoints from ever
+    # receiving the official OpenAI credential by accident.
+    class OpenAICompatible < OpenAI
+      DEFAULT_MODEL = nil
+
+      def initialize(endpoint:)
+        super(
+          api_url: endpoint,
+          api_key: ENV.fetch('OPENAI_COMPATIBLE_API_KEY', nil),
+          require_api_key: false
+        )
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/llm/prompt_evidence.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  module LLM
+    # Converts application and supplemental inputs into prompt-safe evidence records.
+    module PromptEvidence
+      private
+
+      def detect_platform(matches)
+        return 'mobile' if matches.empty?
+
+        platforms = matches.filter_map { |match| FileClassifier.searchable_platform(match.file) }
+
+        if platforms.include?(:ios)
+          'iOS'
+        elsif platforms.include?(:android)
+          'Android'
+        else
+          'mobile'
+        end
+      end
+
+      def prompt_matches(matches, include_file_paths:, redact_prompts:)
+        matches.map do |match|
+          location = include_file_paths ? match.file : File.basename(match.file)
+          {
+            location: sanitized_prompt_value(location, redact: redact_prompts),
+            line: match.line,
+            enclosing_scope: sanitized_prompt_value(match.enclosing_scope, redact: redact_prompts),
+            matched_line: sanitized_prompt_value(match.match_line, redact: redact_prompts),
+            context: sanitized_prompt_value(match.context, redact: redact_prompts)
+          }.compact
+        end
+      end
+
+      def prompt_context_sources(sources, redact_prompts:)
+        sources.map do |source|
+          {
+            kind: sanitized_prompt_value(source.kind, redact: redact_prompts),
+            name: sanitized_prompt_value(source.name, redact: redact_prompts),
+            content: sanitized_prompt_value(source.content, redact: redact_prompts)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/llm/request_policy.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  module LLM
+    # Shared retry and HTTP error handling for remote LLM providers.
+    module RequestPolicy
+      RequestOutcome = Data.define(:response, :retries) do
+        def request_count = retries + 1
+      end
+
+      # Adds run-local attempt counters to a transport exception without
+      # changing its public exception class.
+      module FailureTelemetry
+        attr_reader :request_count, :retries
+
+        def record_request_failure(retries:)
+          @retries = retries
+          @request_count = retries + 1
+          self
+        end
+      end
+      private_constant :FailureTelemetry
+
+      MAX_RETRIES = 2
+      MAX_RETRY_DELAY = 30.0
+      RETRYABLE_STATUS_CODES = [408, 409, 425, 429, 500, 502, 503, 504, 529].freeze
+      TRANSIENT_NETWORK_ERRORS = [
+        Net::OpenTimeout,
+        Net::ReadTimeout,
+        Timeout::Error,
+        EOFError,
+        SocketError,
+        Errno::ECONNABORTED,
+        Errno::ECONNREFUSED,
+        Errno::ECONNRESET,
+        Errno::EHOSTUNREACH,
+        Errno::ENETUNREACH,
+        Errno::ETIMEDOUT,
+        OpenSSL::SSL::SSLError
+      ].freeze
+
+      protected
+
+      def request_with_retries(uri:)
+        retries = 0
+
+        loop do
+          response = yield
+          return RequestOutcome.new(response: response, retries: retries) unless retryable_response?(response) && retries < MAX_RETRIES
+
+          retries += 1
+          delay = retry_delay(response, retries)
+          reset_http_session(uri)
+          sleep(delay)
+        rescue *TRANSIENT_NETWORK_ERRORS => e
+          if retries >= MAX_RETRIES
+            e.extend(FailureTelemetry).record_request_failure(retries: retries)
+            raise
+          end
+
+          retries += 1
+          reset_http_session(uri)
+          sleep(retry_delay(nil, retries))
+          retry
+        end
+      end
+
+      def retryable_response?(response)
+        status = response.code.to_i
+        RETRYABLE_STATUS_CODES.include?(status)
+      end
+
+      def retry_delay(response, retry_number)
+        retry_after = parse_retry_after(response&.[]('retry-after'))
+        return [retry_after, MAX_RETRY_DELAY].min if retry_after
+
+        base = 0.5 * (2**(retry_number - 1))
+        [base + (rand * 0.25), MAX_RETRY_DELAY].min
+      end
+
+      def parse_retry_after(value)
+        return nil if value.nil? || value.to_s.strip.empty?
+
+        numeric = Float(value, exception: false)
+        return [numeric, 0].max if numeric&.finite?
+
+        [Time.httpdate(value) - Time.now, 0].max
+      rescue ArgumentError
+        nil
+      end
+
+      def http_error_result(response, retries: 0)
+        telemetry = request_telemetry(retries: retries)
+        case response.code.to_i
+        when 401
+          ContextResult.new(
+            description: 'Authentication failed',
+            error: 'Provider rejected the API credentials',
+            **telemetry
+          )
+        when 429
+          ContextResult.new(
+            description: 'Rate limited',
+            error: 'Rate limit exceeded - try reducing concurrency',
+            **telemetry
+          )
+        else
+          ContextResult.new(description: 'API error', error: provider_error_message(response), **telemetry)
+        end
+      end
+
+      def provider_error_message(response)
+        body = JSON.parse(response.body)
+        return "HTTP #{response.code}" unless body.is_a?(Hash)
+
+        message = body.dig('error', 'message') || body['message']
+        return "HTTP #{response.code}" unless message.is_a?(String) && !message.empty?
+
+        message.gsub(/[\u0000-\u001F\u007F]/, ' ')[0, 500]
+      rescue JSON::ParserError, TypeError
+        "HTTP #{response.code}"
+      end
+
+      def request_telemetry(retries:)
+        { request_count: retries + 1, retries: retries }
+      end
+
+      def failure_request_telemetry(error, outcome:)
+        return request_telemetry(retries: outcome.retries) if outcome
+        return { request_count: error.request_count, retries: error.retries } if
+          error.respond_to?(:request_count) && error.respond_to?(:retries)
+
+        {}
+      end
+
+      def reset_http_session(uri)
+        sessions = Thread.current.thread_variable_get(http_sessions_key)
+        return unless sessions
+
+        http = sessions.delete([uri.scheme, uri.host, uri.port])
+        http.finish if http&.started?
+      rescue IOError, SystemCallError
+        nil
+      end
+    end
+  end
+end

data/lib/i18n_context_generator/localization_syntax.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+require_relative 'android_resource'
+require_relative 'apple_string_literal'
+
+module I18nContextGenerator
+  # Registry of localization call/resource syntaxes shared by source search,
+  # source-first discovery, and Swift comment write-back.
+  class LocalizationSyntax
+    DEFAULT_SWIFT_FUNCTIONS = %w[
+      NSLocalizedString
+      String(localized:
+      Text(
+      LocalizedStringResource(
+    ].freeze
+
+    IOS_STATIC_SEARCH_BUILDERS = [
+      ->(key) { "LocalizedStringKey\\s*\\(\\s*\"#{key}\"" },
+      ->(key) { "LocalizedStringKey\\s*=\\s*\"#{key}\"" },
+      ->(key) { ":\\s*LocalizedStringResource\\s*=\\s*\"#{key}\"" },
+      ->(key) { "\"#{key}\"\\.localized" }
+    ].freeze
+    SWIFT_STRING_BODY_PATTERN = AppleStringLiteral::BODY_PATTERN
+    private_constant :SWIFT_STRING_BODY_PATTERN
+
+    IOS_STATIC_SINGLE_LINE_DISCOVERY_PATTERNS = [
+      /LocalizedStringKey\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\s*\)/,
+      /:\s*LocalizedStringKey\s*=\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/,
+      /:\s*LocalizedStringResource\s*=\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/,
+      /"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\.localized\b/
+    ].freeze
+
+    LOCALIZED_RESOURCE_SINGLE_LINE_DISCOVERY_PATTERNS = [
+      /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"[^\n]*?\bdefaultValue:\s*"(?<text>#{SWIFT_STRING_BODY_PATTERN})"/,
+      /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/
+    ].freeze
+
+    IOS_STATIC_MULTILINE_DISCOVERY_PATTERNS = [
+      /Text\s*\(\s*LocalizedStringKey\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\s*\)[\s\S]*?\)/
+    ].freeze
+
+    LOCALIZED_RESOURCE_MULTILINE_DISCOVERY_PATTERNS = [
+      /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"[\s\S]*?\bdefaultValue:\s*"(?<text>#{SWIFT_STRING_BODY_PATTERN})"/,
+      /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/
+    ].freeze
+
+    OPTIONAL_ARGUMENT_LABEL_PATTERN = '(?:[A-Za-z_]\w*\s*:\s*)?'
+    private_constant :OPTIONAL_ARGUMENT_LABEL_PATTERN
+
+    ANDROID_DISCOVERY_PATTERNS = {
+      string: %r{R\.string\.(\w+)\b|@string/([\w.]+)\b|[(\s,=]string\.(\w+)\b}x,
+      plural: %r{R\.plurals\.(\w+)\b|@plurals/([\w.]+)\b|[(\s,=]plurals\.(\w+)\b}x,
+      array: %r{R\.array\.(\w+)\b|@array/([\w.]+)\b|[(\s,=]array\.(\w+)\b}x
+    }.freeze
+
+    def self.functions_with_defaults(functions)
+      configured = Array(functions).map(&:strip).reject(&:empty?)
+      (DEFAULT_SWIFT_FUNCTIONS + configured).uniq { |function| function.sub(/\(\s*\z/, '') }.freeze
+    end
+
+    def self.swift_string_content_pattern(value)
+      encoded = value.each_char.map do |character|
+        case character
+        when '\\' then '\\\\'
+        when '"' then '\\"'
+        when "\n" then '\n'
+        when "\r" then '\r'
+        when "\t" then '\t'
+        else character
+        end
+      end.join
+      Regexp.escape(encoded)
+    end
+
+    def initialize(swift_functions: nil)
+      @swift_functions = self.class.functions_with_defaults(swift_functions)
+    end
+
+    attr_reader :swift_functions
+
+    def search_patterns(key, platform:, resource_type: nil)
+      strings = case platform.to_sym
+                when :ios then ios_search_patterns(key)
+                when :android then android_search_patterns(key, resource_type: resource_type)
+                else
+                  ios_search_patterns(key) + android_search_patterns(key, resource_type: resource_type) +
+                  [Regexp.escape(key)]
+                end
+      strings.map { |pattern| Regexp.new(pattern) }
+    end
+
+    def ios_search_patterns(key)
+      escaped_key = self.class.swift_string_content_pattern(key)
+      function_patterns = swift_function_key_patterns(escaped_key)
+      function_patterns + IOS_STATIC_SEARCH_BUILDERS.map { |builder| builder.call(escaped_key) }
+    end
+
+    def ios_multiline_search_patterns(key)
+      escaped_key = self.class.swift_string_content_pattern(key)
+      swift_function_key_patterns(escaped_key).map { |pattern| Regexp.new(pattern) }
+    end
+
+    def android_search_patterns(key, resource_type: nil)
+      base_key = AndroidResource.base_key(key)
+      escaped_base = Regexp.escape(base_key)
+
+      case AndroidResource.type_for(key, explicit: resource_type)
+      when :plural then android_plural_patterns(escaped_base)
+      when :array then android_array_patterns(escaped_base)
+      else android_string_patterns(Regexp.escape(key))
+      end
+    end
+
+    def ios_single_line_discovery_patterns
+      @ios_single_line_discovery_patterns ||=
+        (LOCALIZED_RESOURCE_SINGLE_LINE_DISCOVERY_PATTERNS +
+          function_discovery_patterns(multiline: false) +
+          IOS_STATIC_SINGLE_LINE_DISCOVERY_PATTERNS).freeze
+    end
+
+    def ios_multiline_discovery_patterns
+      @ios_multiline_discovery_patterns ||=
+        (LOCALIZED_RESOURCE_MULTILINE_DISCOVERY_PATTERNS +
+          function_discovery_patterns(multiline: true) +
+          IOS_STATIC_MULTILINE_DISCOVERY_PATTERNS).freeze
+    end
+
+    def ios_call_start_patterns
+      @ios_call_start_patterns ||=
+        @swift_functions.map { |function| Regexp.new(swift_function_opener(function)) }.freeze
+    end
+
+    def ios_wrapper_definition_pattern
+      @ios_wrapper_definition_pattern ||= begin
+        functions = (@swift_functions.map { |function| swift_call_prefix(function) } +
+          ['LocalizedStringKey\\s*\\(']).join('|')
+        constructor = "\\s*=\\s*(?:#{functions})"
+        typed_resource = '\s*:\s*LocalizedStringResource\s*=\s*"'
+        /\bstatic\s+(?:let|var)\s+(?<member_name>\w+)(?:#{constructor}|#{typed_resource})/
+      end
+    end
+
+    def android_discovery_patterns
+      ANDROID_DISCOVERY_PATTERNS
+    end
+
+    def swift_writer_patterns(key)
+      escaped_key = self.class.swift_string_content_pattern(key)
+      comment = 'comment:\\s*"(?:\\\\.|[^"\\\\])*"'
+
+      patterns = @swift_functions.map do |function|
+        Regexp.new(
+          "#{swift_key_argument_prefix(function)}\"#{escaped_key}\"[^)]*#{comment}[^)]*\\)",
+          Regexp::MULTILINE
+        )
+      end
+      patterns << nested_text_writer_pattern(escaped_key, comment)
+      patterns
+    end
+
+    private
+
+    def function_discovery_patterns(multiline:)
+      tail = multiline ? '[\\s\\S]*?' : '[^\\n]*?'
+      @swift_functions.flat_map do |function|
+        prefix = swift_key_argument_prefix(function)
+        [
+          Regexp.new("#{prefix}\\s*@?\"(?<key>#{SWIFT_STRING_BODY_PATTERN})\"#{tail}comment:\\s*\"(?<comment>#{SWIFT_STRING_BODY_PATTERN})\""),
+          Regexp.new("#{prefix}\\s*@?\"(?<key>#{SWIFT_STRING_BODY_PATTERN})\"")
+        ]
+      end
+    end
+
+    def swift_function_key_patterns(escaped_key)
+      @swift_functions.map do |function|
+        "#{swift_key_argument_prefix(function)}@?\"#{escaped_key}\""
+      end
+    end
+
+    def swift_key_argument_prefix(function)
+      prefix = swift_call_prefix(function)
+      return "#{prefix}\\s*" if explicit_argument_fragment?(function) || DEFAULT_SWIFT_FUNCTIONS.include?(function)
+
+      "#{prefix}\\s*#{OPTIONAL_ARGUMENT_LABEL_PATTERN}"
+    end
+
+    def swift_call_prefix(function)
+      name, arguments = function.split('(', 2)
+      prefix = "#{Regexp.escape(name.strip)}\\s*\\("
+      return prefix unless arguments
+
+      arguments = arguments.strip
+      arguments.empty? ? prefix : "#{prefix}\\s*#{Regexp.escape(arguments)}"
+    end
+
+    def swift_function_opener(function)
+      name = function.split('(', 2).first
+      "#{Regexp.escape(name.strip)}\\s*\\("
+    end
+
+    def explicit_argument_fragment?(function)
+      _name, arguments = function.split('(', 2)
+      arguments && !arguments.strip.empty?
+    end
+
+    def nested_text_writer_pattern(escaped_key, comment)
+      Regexp.new(
+        "Text\\s*\\(\\s*LocalizedStringKey\\s*\\(\\s*\"#{escaped_key}\"\\s*\\)" \
+        "[^)]*#{comment}[^)]*\\)",
+        Regexp::MULTILINE
+      )
+    end
+
+    def android_plural_patterns(key)
+      [
+        "R\\.plurals\\.#{key}\\b", "@plurals/#{key}\\b",
+        "getQuantityString\\s*\\(\\s*R\\.plurals\\.#{key}",
+        "\\.getQuantityString\\s*\\(\\s*R\\.plurals\\.#{key}",
+        "pluralStringResource\\s*\\(\\s*R\\.plurals\\.#{key}",
+        "[\\(\\s,=]plurals\\.#{key}\\b"
+      ]
+    end
+
+    def android_array_patterns(key)
+      [
+        "R\\.array\\.#{key}\\b", "@array/#{key}\\b",
+        "getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
+        "\\.getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
+        "resources\\.getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
+        "[\\(\\s,=]array\\.#{key}\\b"
+      ]
+    end
+
+    def android_string_patterns(key)
+      [
+        "R\\.string\\.#{key}\\b", "@string/#{key}\\b",
+        "getString\\s*\\(\\s*R\\.string\\.#{key}",
+        "\\.getString\\s*\\(\\s*R\\.string\\.#{key}",
+        "stringResource\\s*\\(\\s*R\\.string\\.#{key}",
+        "[\\(\\s,=]string\\.#{key}\\b",
+        "getString\\s*\\(\\s*string\\.#{key}",
+        "stringResource\\s*\\(\\s*string\\.#{key}"
+      ]
+    end
+  end
+end

data/lib/i18n_context_generator/parsers/android_xml_parser.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'rexml/document'
+require_relative '../android_resource'
 
 module I18nContextGenerator
   module Parsers
@@ -11,6 +12,7 @@ module I18nContextGenerator
       def parse(path)
         content = File.read(path, encoding: 'UTF-8')
         doc = REXML::Document.new(content)
+        resource_index = AndroidResource.index(content)
         entries = []
 
         doc.elements.each('resources/string') do |element|
@@ -22,11 +24,12 @@ module I18nContextGenerator
           # Look for preceding comment
           comment = find_preceding_comment(element)
 
-          entries << TranslationEntry.new(
+          entries << build_entry(
             key: key,
             text: unescape_android_string(text),
             source_file: path,
-            metadata: { comment: comment }
+            metadata: { comment: comment, resource_type: :string },
+            resource_index: resource_index
           )
         end
 
@@ -35,12 +38,22 @@ module I18nContextGenerator
           next unless translatable?(array_element)
 
           array_name = array_element.attributes['name']
+          parent_comment = find_preceding_comment(array_element)
           array_element.elements.to_a('item').each_with_index do |item, index|
-            entries << TranslationEntry.new(
-              key: "#{array_name}[#{index}]",
+            key = AndroidResource.composite_key(array_name, type: :array, index: index)
+            item_comment = find_preceding_comment(item)
+            entries << build_entry(
+              key: key,
               text: unescape_android_string(inner_text(item)),
               source_file: path,
-              metadata: { array: array_name, index: index }
+              metadata: {
+                array: array_name,
+                index: index,
+                resource_type: :array,
+                parent_comment: parent_comment,
+                comment: combined_comment(parent_comment, item_comment)
+              }.compact,
+              resource_index: resource_index
             )
           end
         end
@@ -50,22 +63,44 @@ module I18nContextGenerator
           next unless translatable?(plural_element)
 
           plural_name = plural_element.attributes['name']
+          parent_comment = find_preceding_comment(plural_element)
           plural_element.elements.each('item') do |item|
             quantity = item.attributes['quantity']
-            entries << TranslationEntry.new(
-              key: "#{plural_name}:#{quantity}",
+            key = AndroidResource.composite_key(plural_name, type: :plural, quantity: quantity)
+            item_comment = find_preceding_comment(item)
+            entries << build_entry(
+              key: key,
               text: unescape_android_string(inner_text(item)),
               source_file: path,
-              metadata: { plural: plural_name, quantity: quantity }
+              metadata: {
+                plural: plural_name,
+                quantity: quantity,
+                resource_type: :plural,
+                parent_comment: parent_comment,
+                comment: combined_comment(parent_comment, item_comment)
+              }.compact,
+              resource_index: resource_index
             )
           end
         end
 
         entries
+      rescue REXML::ParseException => e
+        raise Error, "Failed to parse Android XML translation file #{path}: #{e.message.lines.first&.strip}"
       end
 
       private
 
+      def build_entry(key:, text:, source_file:, metadata:, resource_index:)
+        span = resource_index.span_for(key)
+        if span
+          metadata = metadata.merge(
+            line_span: span.line_span
+          )
+        end
+        TranslationEntry.new(key: key, text: text, source_file: source_file, metadata: metadata)
+      end
+
       # Get the full inner content of an element, including inline markup like
       # <b>, <i>, <u>, <xliff:g>. REXML::Element#text only returns the first
       # text node, losing everything after a nested element.
@@ -91,6 +126,10 @@ module I18nContextGenerator
         nil
       end
 
+      def combined_comment(parent_comment, item_comment)
+        [parent_comment, item_comment].compact.uniq.join("\n").then { |comment| comment unless comment.empty? }
+      end
+
       def translatable?(element)
         element.attributes['translatable']&.downcase != 'false'
       end

data/lib/i18n_context_generator/parsers/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../file_classifier'
+
 module I18nContextGenerator
   module Parsers
     # Represents a single translation entry
@@ -11,20 +13,21 @@ module I18nContextGenerator
 
     # Base class for translation file parsers
     class Base
-      def self.for(path)
-        basename = File.basename(path).downcase
+      def self.for(path, locale: nil)
         ext = File.extname(path).downcase
 
         case ext
         when '.json'
           JsonParser.new
         when '.yml', '.yaml'
-          YamlParser.new
+          YamlParser.new(locale: locale)
         when '.strings'
           StringsParser.new
+        when '.xcstrings'
+          XcstringsParser.new
         when '.xml'
           # Check if it's an Android strings.xml
-          raise Error, "Unsupported XML format: #{path} (only Android strings.xml is supported)" unless basename == 'strings.xml' || path.include?('/res/values')
+          raise Error, "Unsupported XML format: #{path} (only Android strings.xml is supported)" unless FileClassifier.android_translation_file?(path)
 
           AndroidXmlParser.new
 

data/lib/i18n_context_generator/parsers/json_parser.rb

@@ -6,6 +6,8 @@ module I18nContextGenerator
     class JsonParser < Base
       def parse(path)
         data = Oj.load_file(path)
+        raise Error, "Invalid JSON translation file #{path}: root must be an object" unless data.is_a?(Hash)
+
         flatten_keys(data).filter_map do |key, text|
           next if text.nil? || text.to_s.strip.empty?
 
@@ -15,6 +17,8 @@ module I18nContextGenerator
             source_file: path
           )
         end
+      rescue Oj::ParseError => e
+        raise Error, "Failed to parse JSON translation file #{path}: #{e.message}"
       end
     end
   end