i18n-context-generator 0.4.0 → 0.5.1

data/lib/i18n_context_generator/git_diff.rb

@@ -2,30 +2,74 @@
 
 require 'open3'
 require 'pathname'
+require_relative 'apple_string_literal'
+require_relative 'changed_location'
+require_relative 'translation_comment_index'
+require_relative 'xml_scanner'
+require_relative 'android_resource'
+require_relative 'xcstrings_document'
+require_relative 'git_diff/xml_changes'
 
 module I18nContextGenerator
   # Parses git diff to extract changed translation keys
   class GitDiff
-    def initialize(base_ref: 'main')
+    include GitDiffXmlChanges
+
+    def initialize(base_ref: 'main', head_ref: 'HEAD')
       @base_ref = base_ref
+      @head_ref = head_ref
     end
 
     # Get keys that were added or modified since the base ref
     # @param translation_paths [Array<String>] paths to translation files
     # @return [Set<String>] set of changed keys
     def changed_keys(translation_paths)
-      keys = Set.new
+      changed_key_locations(translation_paths).each_key.to_set(&:last)
+    end
 
-      translation_paths.each do |path|
+    # Get changed translation keys together with the exact changed lines that
+    # produced them. Keys are scoped by translation file so duplicate keys in
+    # different files remain distinct.
+    # @return [Hash{Array(String, String) => Array<ChangedLocation>}]
+    def changed_key_locations(translation_paths)
+      translation_paths.each_with_object({}) do |path, changes|
         next unless File.exist?(path)
 
         diff_output = git_diff_for_path(path)
         next if diff_output.empty?
 
-        keys.merge(extract_keys_from_diff(diff_output, path))
+        normalized_path = Pathname.new(path).cleanpath.to_s
+        base_content, head_content = revision_contents(path) if revision_contents_needed?(path, diff_output)
+        head_content ||= File.binread(path)
+        key_locations = case File.extname(path).downcase
+                        when '.strings'
+                          extract_strings_key_locations(
+                            diff_output,
+                            normalized_path,
+                            base_content: base_content,
+                            head_content: head_content
+                          )
+                        when '.xcstrings'
+                          extract_xcstrings_key_locations(
+                            diff_output,
+                            normalized_path,
+                            base_content: base_content,
+                            head_content: head_content
+                          )
+                        when '.xml'
+                          extract_xml_key_locations(
+                            diff_output,
+                            normalized_path,
+                            base_content: base_content,
+                            head_content: head_content
+                          )
+                        else
+                          {}
+                        end
+        key_locations.each do |key, locations|
+          changes[[normalized_path, key]] = locations
+        end
       end
-
-      keys
     end
 
     # Get changed line numbers in source files since the base ref.
@@ -52,35 +96,56 @@ module I18nContextGenerator
       system('git', 'rev-parse', '--verify', @base_ref, out: File::NULL, err: File::NULL)
     end
 
+    def head_ref_exists?
+      system('git', 'rev-parse', '--verify', @head_ref, out: File::NULL, err: File::NULL)
+    end
+
     private
 
     def git_diff_for_path(path)
       # Run git from the directory containing the file so the correct repo is used
       dir = File.directory?(path) ? path : File.dirname(path)
       pathspec = File.directory?(path) ? '.' : File.basename(path)
-      # Use triple-dot to get changes on current branch since it diverged from base
-      stdout, _stderr, status = Open3.capture3('git', 'diff', "#{@base_ref}...HEAD", '--', pathspec, chdir: dir)
-      status.success? ? stdout : ''
+      # Use triple-dot to get changes on the configured head since it diverged from base.
+      stdout, stderr, status = Open3.capture3(
+        'git', 'diff', "#{@base_ref}...#{@head_ref}", '--', pathspec, chdir: dir
+      )
+      return stdout if status.success?
+
+      detail = stderr.strip
+      detail = 'git exited unsuccessfully without an error message' if detail.empty?
+      raise Error, "Git diff failed for #{@base_ref}...#{@head_ref} (#{path}): #{detail}"
+    rescue SystemCallError => e
+      raise Error, "Git diff failed for #{@base_ref}...#{@head_ref} (#{path}): #{e.message}"
     end
 
     def extract_changed_lines(diff_output, path)
       changed_lines = Hash.new { |h, k| h[k] = Set.new }
       current_file = File.file?(path) ? path : nil
       file_line = nil
+      in_hunk = false
 
       diff_output.each_line do |line|
-        if (match = line.match(%r{^\+\+\+ b/(.+)$}))
+        if line.start_with?('diff ')
+          file_line = nil
+          in_hunk = false
+          next
+        end
+
+        if !in_hunk && (match = line.match(%r{^\+\+\+ b/(.+)$}))
           current_file = resolve_diff_file_path(path, match[1])
           next
         end
 
         if (hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/))
           file_line = hunk[1].to_i
+          in_hunk = true
           next
         end
 
-        next if line.start_with?('diff ', 'index ', '--- ', '+++ ', '\\')
-        next if file_line.nil? || current_file.nil?
+        next if !in_hunk && line.start_with?('index ', '--- ', '+++ ')
+        next if line.start_with?('\\')
+        next if !in_hunk || file_line.nil? || current_file.nil?
 
         if line.start_with?('+')
           changed_lines[current_file] << file_line
@@ -98,124 +163,188 @@ module I18nContextGenerator
     def resolve_diff_file_path(path, diff_file_path)
       return Pathname.new(path).cleanpath.to_s if File.file?(path)
 
-      normalized_path = path.to_s.sub(%r{/\z}, '')
+      normalized_path = Pathname.new(path).cleanpath.to_s
+      if Pathname.new(normalized_path).absolute?
+        prefix = repository_prefix_for(path)
+        relative_path = diff_file_path.delete_prefix(prefix)
+        return Pathname.new(File.join(normalized_path, relative_path)).cleanpath.to_s
+      end
       return Pathname.new(diff_file_path).cleanpath.to_s if normalized_path.empty? || normalized_path == '.'
       return Pathname.new(diff_file_path).cleanpath.to_s if diff_file_path == normalized_path || diff_file_path.start_with?("#{normalized_path}/")
 
       Pathname.new(File.join(normalized_path, diff_file_path)).cleanpath.to_s
     end
 
+    def repository_prefix_for(path)
+      @repository_prefixes ||= {}
+      directory = File.directory?(path) ? path : File.dirname(path)
+      @repository_prefixes[directory] ||= begin
+        stdout, stderr, status = Open3.capture3('git', 'rev-parse', '--show-prefix', chdir: directory)
+        if status.success?
+          stdout.strip.sub(%r{/\z}, '')
+        else
+          detail = stderr.strip
+          detail = 'unable to resolve repository path prefix' if detail.empty?
+          raise Error, "Git diff failed for #{@base_ref}...#{@head_ref} (#{path}): #{detail}"
+        end
+      end
+    end
+
     def merge_line_maps!(target, source)
       source.each do |file, lines|
         target[file].merge(lines)
       end
     end
 
-    def extract_keys_from_diff(diff_output, path)
-      ext = File.extname(path).downcase
-
-      case ext
-      when '.strings'
-        extract_strings_keys(diff_output)
-      when '.xml'
-        extract_xml_keys(diff_output, path)
-      else
-        Set.new
+    def extract_strings_key_locations(diff_output, file_path, base_content: nil, head_content: nil)
+      locations = Hash.new { |hash, key| hash[key] = [] }
+      head_content ||= File.binread(file_path)
+      head_index = TranslationCommentIndex.new(format: :strings, content: head_content)
+      base_index = TranslationCommentIndex.new(format: :strings, content: base_content) if base_content
+
+      each_changed_diff_line(diff_output) do |content, old_line, new_line, side|
+        if side == :right
+          key = AppleStringLiteral.assignment_key(content) || head_index.key_at(new_line)
+          locations[key] << changed_location(file_path, new_line, side: :right) if key
+        elsif base_index
+          key = AppleStringLiteral.assignment_key(content) || base_index.key_at(old_line)
+          fallback_line = head_index.line_for_key(key)
+          if key
+            locations[key] << changed_location(
+              file_path,
+              old_line,
+              side: :left,
+              fallback_line: fallback_line
+            )
+          end
+        end
       end
+
+      prefer_right_locations(locations)
     end
 
-    # Extract keys from iOS .strings diff
-    # Looks for added lines like: +"key" = "value";
-    def extract_strings_keys(diff_output)
-      keys = Set.new
+    def extract_xcstrings_key_locations(diff_output, file_path, base_content: nil, head_content: nil)
+      current_content = head_content || File.binread(file_path)
+      head_index = xcstrings_line_index(head_content || current_content, file_path)
+      base_index = xcstrings_line_index(base_content || head_content || current_content, file_path)
+      locations = Hash.new { |hash, key| hash[key] = [] }
+      head_lines = first_lines_by_key(head_index)
 
-      diff_output.each_line do |line|
-        # Match added or modified lines (start with +, not ++)
-        next unless line.start_with?('+') && !line.start_with?('++')
+      each_changed_diff_line(diff_output) do |content, old_line, new_line, side|
+        next if content.strip.empty?
 
-        # Extract key from: "key" = "value";
-        keys << Regexp.last_match(1) if line =~ /^\+\s*"([^"]+)"\s*=/
+        if side == :right
+          key = head_index[new_line]
+          locations[key] << changed_location(file_path, new_line, side: :right) if key
+        else
+          key = base_index[old_line]
+          fallback_line = head_lines[key]
+          if key
+            locations[key] << changed_location(
+              file_path,
+              old_line,
+              side: :left,
+              fallback_line: fallback_line
+            )
+          end
+        end
       end
 
-      keys
+      prefer_right_locations(locations)
     end
 
-    # Extract keys from Android strings.xml diff.
-    # Tracks parent element context from diff lines and uses hunk headers to
-    # map added lines to file positions. When an added <item> can't be attributed
-    # to a parent from diff context alone (e.g. large plural/array blocks where the
-    # opener isn't in the hunk), falls back to reading the actual file.
-    def extract_xml_keys(diff_output, file_path)
-      keys = Set.new
-      current_parent = nil
-      file_line = nil
-      orphaned_item_file_lines = []
+    def xcstrings_line_index(content, file_path)
+      return {} unless content
 
-      diff_output.each_line do |line|
-        # Parse hunk header to track position in new file
-        if (hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/))
-          file_line = hunk[1].to_i
-          next
-        end
+      XcstringsDocument.new(content, path: file_path).line_index
+    end
 
-        # Skip diff metadata lines
-        next if line.start_with?('diff ', 'index ', '--- ', '+++ ')
+    def revision_contents_needed?(path, diff_output)
+      File.extname(path).downcase == '.xcstrings' ||
+        diff_output.each_line.any? { |line| line.start_with?('-') && !line.start_with?('---') }
+    end
 
-        is_removed = line.start_with?('-')
-        is_added = line.start_with?('+')
-        content = line.sub(/^[ +-]/, '')
+    def revision_contents(path)
+      directory = File.dirname(File.expand_path(path))
+      root, stderr, status = Open3.capture3('git', 'rev-parse', '--show-toplevel', chdir: directory)
+      unless status.success?
+        detail = stderr.strip
+        detail = 'unable to resolve repository root' if detail.empty?
+        raise Error, "Git diff failed for #{@base_ref}...#{@head_ref} (#{path}): #{detail}"
+      end
 
-        # Track parent element from any visible line (context, added, or removed)
-        if content =~ /<(?:plurals|string-array)\s+name=["']([^"']+)["']/
-          current_parent = Regexp.last_match(1)
-        elsif content =~ %r{</(?:plurals|string-array)>}
-          current_parent = nil
-        end
+      repository_root = root.strip
+      relative_path = Pathname.new(File.expand_path(path))
+                              .relative_path_from(Pathname.new(repository_root)).to_s
+      merge_base, _stderr, status = Open3.capture3(
+        'git', 'merge-base', @base_ref, @head_ref, chdir: repository_root
+      )
+      base_revision = status.success? && !merge_base.strip.empty? ? merge_base.strip : @base_ref
+      [
+        file_at_revision(repository_root, relative_path, base_revision),
+        file_at_revision(repository_root, relative_path, @head_ref)
+      ]
+    end
 
-        # Process added lines for key extraction
-        if is_added
-          keys << Regexp.last_match(1) if content =~ /<string\s+name=["']([^"']+)["']/
-          keys << Regexp.last_match(1) if content =~ /<(?:plurals|string-array)\s+name=["']([^"']+)["']/
-
-          if content =~ /^\s*<item[\s>]/
-            if current_parent
-              keys << current_parent
-            elsif file_line
-              orphaned_item_file_lines << file_line
-            end
-          end
-        end
+    def file_at_revision(repository_root, relative_path, revision)
+      stdout, _stderr, status = Open3.capture3(
+        'git', 'show', "#{revision}:#{relative_path}", chdir: repository_root
+      )
+      status.success? ? stdout : nil
+    end
 
-        # Context and added lines exist in new file; removed lines do not
-        file_line += 1 if file_line && !is_removed
-      end
+    def changed_location(file, line, side:, fallback_line: nil)
+      ChangedLocation.new(file: file, line: line, side: side, fallback_line: fallback_line)
+    end
 
-      # Resolve orphaned items by reading the actual file
-      resolve_orphaned_items(keys, orphaned_item_file_lines, file_path)
+    def prefer_right_locations(locations)
+      preferred = locations.transform_values do |values|
+        unique = values.uniq
+        right = unique.select(&:right?)
+        right.empty? ? unique : right
+      end
+      preferred.reject { |_key, values| values.empty? }
+    end
 
-      keys
+    def first_lines_by_key(line_index)
+      line_index.each_with_object({}) do |(line, key), lines|
+        lines[key] ||= line
+      end
     end
 
-    # Build a map of file line numbers to enclosing plural/array resource names,
-    # then use it to attribute orphaned <item> additions to their parent.
-    def resolve_orphaned_items(keys, orphaned_lines, file_path)
-      return if orphaned_lines.empty? || !File.exist?(file_path)
+    def each_changed_diff_line(diff_output)
+      old_line_number = nil
+      new_line_number = nil
+      in_hunk = false
 
-      current_parent = nil
-      parent_at_line = {}
+      diff_output.each_line do |line|
+        if line.start_with?('diff ')
+          old_line_number = nil
+          new_line_number = nil
+          in_hunk = false
+          next
+        end
 
-      File.readlines(file_path).each_with_index do |line, index|
-        if line =~ /<(?:plurals|string-array)\s+name=["']([^"']+)["']/
-          current_parent = Regexp.last_match(1)
-        elsif line =~ %r{</(?:plurals|string-array)>}
-          current_parent = nil
+        if (match = line.match(/^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/))
+          old_line_number = match[1].to_i
+          new_line_number = match[2].to_i
+          in_hunk = true
+          next
         end
-        parent_at_line[index + 1] = current_parent
-      end
+        next if !in_hunk && line.start_with?('index ', '--- ', '+++ ')
+        next if line.start_with?('\\')
+        next if !in_hunk || old_line_number.nil? || new_line_number.nil?
 
-      orphaned_lines.each do |line_num|
-        parent = parent_at_line[line_num]
-        keys << parent if parent
+        if line.start_with?('+')
+          yield(line[1..], nil, new_line_number, :right)
+          new_line_number += 1
+        elsif line.start_with?('-')
+          yield(line[1..], old_line_number, nil, :left)
+          old_line_number += 1
+        else
+          old_line_number += 1
+          new_line_number += 1
+        end
       end
     end
   end

data/lib/i18n_context_generator/llm/anthropic.rb

@@ -2,7 +2,7 @@
 
 module I18nContextGenerator
   module LLM
-    # Claude API implementation of the LLM client with retry logic for rate limits.
+    # Claude API implementation of the LLM client.
     class Anthropic < Client
       API_URL = 'https://api.anthropic.com/v1/messages'
       ANTHROPIC_VERSION = '2023-06-01'
@@ -16,36 +16,31 @@ module I18nContextGenerator
         @uri = URI(API_URL)
       end
 
-      MAX_RETRIES = 2
-
       def generate_context(key:, text:, matches:, model: nil, comment: nil,
-                           include_file_paths: false, redact_prompts: true)
-        model ||= DEFAULT_MODEL
+                           include_file_paths: false, redact_prompts: true,
+                           max_prompt_chars: nil, supplemental_context: [])
+        outcome = nil
+        model = resolved_model(model)
         prompt = build_prompt(
           key: key,
           text: text,
           matches: matches,
           comment: comment,
           include_file_paths: include_file_paths,
-          redact_prompts: redact_prompts
+          redact_prompts: redact_prompts,
+          max_prompt_chars: max_prompt_chars,
+          supplemental_context: supplemental_context
         )
-        retries = 0
-
-        loop do
-          response = post_request(model: model, prompt: prompt)
-
-          # Retry on rate limit with backoff
-          if response.code.to_i == 429 && retries < MAX_RETRIES
-            retries += 1
-            delay = (response['retry-after']&.to_i || 2) * retries
-            sleep(delay)
-            next
-          end
-
-          return handle_response(response)
-        end
+        outcome = request_with_retries(uri: @uri) { post_request(model: model, prompt: prompt) }
+        handle_response(outcome.response, retries: outcome.retries)
+      rescue PromptPreparationError => e
+        ContextResult.new(description: 'Prompt preparation failed', error: e.message)
       rescue StandardError => e
-        ContextResult.new(description: 'API request failed', error: e.message)
+        ContextResult.new(
+          description: 'API request failed',
+          error: e.message,
+          **failure_request_telemetry(e, outcome: outcome)
+        )
       end
 
       private
@@ -59,33 +54,65 @@ module I18nContextGenerator
           },
           body: {
             model: model,
-            max_tokens: 500,
+            max_tokens: MAX_OUTPUT_TOKENS,
             system: SYSTEM_PROMPT,
-            messages: [{ role: 'user', content: prompt }]
+            messages: [{ role: 'user', content: prompt }],
+            output_config: {
+              format: {
+                type: 'json_schema',
+                schema: RESPONSE_SCHEMA
+              }
+            }
           }
         )
       end
 
-      def handle_response(response)
+      def handle_response(response, retries:)
         case response.code.to_i
         when 200
           body = JSON.parse(response.body)
-          content = body.dig('content', 0, 'text')
-          parse_response(content)
-        when 429
-          ContextResult.new(description: 'Rate limited', error: 'Rate limit exceeded - try reducing concurrency')
-        when 401
-          ContextResult.new(description: 'Authentication failed', error: 'Invalid API key')
+          handle_successful_response(body, retries: retries)
         else
-          error_body = begin
-            JSON.parse(response.body)
-          rescue StandardError
-            {}
-          end
-          error_msg = error_body.dig('error', 'message') || "HTTP #{response.code}"
-          ContextResult.new(description: 'API error', error: error_msg)
+          http_error_result(response, retries: retries)
         end
       end
+
+      def handle_successful_response(body, retries:)
+        telemetry = usage_telemetry(body, retries: retries)
+        case body['stop_reason']
+        when 'end_turn'
+          content = Array(body['content']).find { |item| item['type'] == 'text' }
+          parse_response(content&.[]('text'), telemetry: telemetry)
+        when 'refusal'
+          ContextResult.new(
+            description: 'Provider refused request',
+            error: 'Anthropic refused to generate context',
+            **telemetry
+          )
+        when 'max_tokens'
+          ContextResult.new(
+            description: 'Incomplete response',
+            error: 'Anthropic response reached max_tokens',
+            **telemetry
+          )
+        else
+          reason = body['stop_reason'] || 'missing'
+          ContextResult.new(
+            description: 'Incomplete response',
+            error: "Unexpected Anthropic stop reason: #{reason}",
+            **telemetry
+          )
+        end
+      end
+
+      def usage_telemetry(body, retries:)
+        {
+          input_tokens: body.dig('usage', 'input_tokens').to_i,
+          output_tokens: body.dig('usage', 'output_tokens').to_i,
+          retries: retries,
+          request_count: retries + 1
+        }
+      end
     end
   end
 end