i18n-context-generator 0.3.0

data/lib/i18n_context_generator/cache.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  # File-based cache for LLM results, keyed by translation key and source context.
+  class Cache
+    CACHE_DIR = '.i18n-context-generator-cache'
+    # Bump this when prompt format, search heuristics, or output schema change
+    CACHE_VERSION = 'v2'
+
+    def initialize(enabled: true)
+      @enabled = enabled
+      FileUtils.mkdir_p(CACHE_DIR) if @enabled && !File.directory?(CACHE_DIR)
+    end
+
+    def get(key, text, context: nil)
+      return nil unless @enabled
+
+      path = cache_path(key, text, context)
+      return nil unless File.exist?(path)
+
+      Oj.load_file(path, symbol_keys: true)
+    rescue StandardError => e
+      warn "Cache read error for #{key}: #{e.message}"
+      nil
+    end
+
+    def set(key, text, result, context: nil)
+      return unless @enabled
+
+      path = cache_path(key, text, context)
+      File.write(path, Oj.dump(result, indent: 2, mode: :compat))
+    rescue StandardError => e
+      warn "Cache write error for #{key}: #{e.message}"
+    end
+
+    def clear
+      FileUtils.rm_rf(CACHE_DIR) if File.directory?(CACHE_DIR)
+    end
+
+    private
+
+    def cache_path(key, text, context)
+      # Include version, key, text, and context (match locations/code) in hash
+      # so cache invalidates when source code usage changes
+      hash = Digest::MD5.hexdigest("#{CACHE_VERSION}:#{key}:#{text}:#{context}")
+      File.join(CACHE_DIR, "#{hash}.json")
+    end
+  end
+end

data/lib/i18n_context_generator/cli.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module I18nContextGenerator
+  # Thor-based CLI entry point for the i18n-context-generator command.
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    desc 'extract', 'Extract translation context from source code'
+    long_desc <<~DESC
+      Analyzes source code to extract contextual information for translation keys.
+      Uses AI to understand how strings are used in the UI and generates descriptions
+      to help translators produce better translations.
+
+      Examples:
+        # iOS app
+        i18n-context-generator extract -t ios/Localizable.strings -s ios/
+
+        # Android app
+        i18n-context-generator extract -t android/res/values/strings.xml -s android/app/
+
+        # Write context back to source files
+        i18n-context-generator extract -t Localizable.strings -s . --write-back
+
+        # Use config file
+        i18n-context-generator extract --config .i18n-context-generator.yml
+    DESC
+    option :config, aliases: '-c', desc: 'Path to config file (.i18n-context-generator.yml)'
+    option :translations, aliases: '-t', desc: 'Translation file(s), comma-separated'
+    option :source, aliases: '-s', desc: 'Source directory(ies) to search, comma-separated'
+    option :output, aliases: '-o', desc: 'Output file path (CSV written only if specified)'
+    option :format, aliases: '-f', enum: %w[csv json], desc: 'Output format (default: csv)'
+    option :provider, aliases: '-p', enum: %w[anthropic openai], desc: 'LLM provider (default: anthropic)'
+    option :model, aliases: '-m', desc: 'LLM model to use'
+    option :keys, aliases: '-k', desc: 'Filter keys (comma-separated patterns, supports * wildcard)'
+    option :concurrency, type: :numeric, desc: 'Number of concurrent requests (default: 5)'
+    option :dry_run, type: :boolean, desc: 'Show what would be processed without calling LLM'
+    option :cache, type: :boolean, desc: 'Enable caching of LLM results'
+    option :write_back, type: :boolean,
+                        desc: 'Write context back to source translation files (.strings, strings.xml)'
+    option :write_back_to_code, type: :boolean,
+                                desc: 'Write context back to Swift source code comment: parameters'
+    option :diff_base, type: :string, desc: 'Only process keys changed since this git ref (e.g., main, origin/main)'
+    option :context_prefix, type: :string,
+                            desc: 'Prefix for context comments (default: "Context: ", use empty string for none)'
+    option :context_mode, type: :string, enum: %w[replace append],
+                          desc: 'How to handle existing comments: replace or append (default: replace)'
+    option :start_key, type: :string, desc: 'Start processing from this key (inclusive)'
+    option :end_key, type: :string, desc: 'Stop processing at this key (inclusive)'
+    option :include_file_paths, type: :boolean,
+                                desc: 'Include full source file paths in LLM prompts (default: false)'
+    option :include_translation_comments, type: :boolean,
+                                          desc: 'Include translation file comments in LLM prompts (default: true)'
+    option :redact_prompts, type: :boolean,
+                            desc: 'Redact likely secrets and PII from LLM prompts (default: true)'
+
+    def extract
+      validate_options!
+      config = Config.load(options)
+      validate_api_key!(provider: config.provider, dry_run: config.dry_run)
+      validate_diff_base!(base_ref: config.diff_base) if config.diff_base
+      extractor = ContextExtractor.new(config)
+      extractor.run
+      fail_if_extraction_errors!(extractor)
+    rescue I18nContextGenerator::Error => e
+      say_error "Error: #{e.message}"
+      exit 1
+    rescue Interrupt
+      say "\nInterrupted"
+      exit 130
+    end
+
+    desc 'init', 'Create a sample config file'
+    option :force, type: :boolean, default: false, desc: 'Overwrite existing config'
+
+    def init
+      config_path = '.i18n-context-generator.yml'
+
+      if File.exist?(config_path) && !options[:force]
+        say_error 'Config file already exists. Use --force to overwrite.'
+        exit 1
+      end
+
+      File.write(config_path, sample_config)
+      say "Created #{config_path}"
+    end
+
+    desc 'version', 'Show version'
+    def version
+      say "i18n-context-generator #{VERSION}"
+    end
+
+    default_task :extract
+
+    private
+
+    def validate_options!
+      return if options[:config] && File.exist?(options[:config])
+
+      return if options[:translations]
+
+      say_error 'Error: --translations (-t) is required unless using a config file'
+      exit 1
+    end
+
+    def validate_api_key!(provider: nil, dry_run: nil)
+      return if dry_run.nil? ? options[:dry_run] : dry_run
+
+      provider ||= options[:provider] || 'anthropic'
+      env_var = case provider
+                when 'anthropic' then 'ANTHROPIC_API_KEY'
+                when 'openai' then 'OPENAI_API_KEY'
+                else "#{provider.upcase}_API_KEY"
+                end
+
+      return if ENV[env_var]
+
+      say_error "Error: #{env_var} environment variable is required for provider '#{provider}'"
+      say_error "Set it with: export #{env_var}=your-api-key"
+      exit 1
+    end
+
+    def validate_diff_base!(base_ref: options[:diff_base])
+      unless GitDiff.available?
+        say_error 'Error: --diff-base requires a git repository'
+        exit 1
+      end
+
+      git_diff = GitDiff.new(base_ref: base_ref)
+      return if git_diff.base_ref_exists?
+
+      say_error "Error: git ref '#{base_ref}' not found"
+      say_error 'Try: origin/main, main, or a specific commit SHA'
+      exit 1
+    end
+
+    def say_error(message)
+      warn message
+    end
+
+    def fail_if_extraction_errors!(extractor)
+      return unless extractor.errors.any?
+
+      say_error "Completed with #{extractor.errors.size} extraction error(s)."
+      exit 1
+    end
+
+    def sample_config
+      <<~YAML
+        # i18n-context-generator configuration
+        # Extract translation context from mobile app source code
+
+        # Translation files to process
+        # Supported formats: .strings (iOS), strings.xml (Android), .json, .yml
+        translations:
+          # iOS example
+          - path: ios/MyApp/Resources/Localizable.strings
+
+          # Android example
+          # - path: android/app/src/main/res/values/strings.xml
+
+        # Source code directories to search
+        source:
+          paths:
+            - ios/MyApp/
+            # - android/app/src/main/java/
+          ignore:
+            - "**/Pods/**"
+            - "**/build/**"
+            - "**/*.generated.*"
+            - "**/*Tests*"
+
+        # LLM configuration
+        llm:
+          provider: anthropic
+          model: claude-sonnet-4-6
+          # API key is read from the matching provider env var
+          # (ANTHROPIC_API_KEY or OPENAI_API_KEY)
+
+        # Processing options
+        processing:
+          concurrency: 5
+          context_lines: 15
+          max_matches_per_key: 3
+
+        # Output configuration
+        output:
+          format: csv
+          path: translation-context.csv
+          # Set to true to write context comments back to translation files (.strings, strings.xml)
+          write_back: false
+          # Set to true to write context back to Swift source code comment: parameters
+          write_back_to_code: false
+          # Prefix for context comments (use empty string for no prefix)
+          # context_prefix: "Context: "
+          # How to handle existing comments: "replace" or "append"
+          # context_mode: replace
+
+        # Swift-specific configuration for write_back_to_code
+        swift:
+          # Localization functions to update (default shown)
+          functions:
+            - NSLocalizedString
+            - "String(localized:"
+            - "Text("
+            # Add custom functions like:
+            # - "MyLocalizedString("
+
+        # Prompt privacy controls
+        privacy:
+          # Include full source paths in prompts sent to the LLM (default: false)
+          include_file_paths: false
+          # Include translation file comments in prompts (default: true)
+          include_translation_comments: true
+          # Redact likely secrets, URLs, and emails before sending prompts (default: true)
+          redact_prompts: true
+      YAML
+    end
+  end
+end

data/lib/i18n_context_generator/config.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module I18nContextGenerator
+  # Holds all configuration for an extraction run, loaded from YAML config files and/or CLI options.
+  class Config
+    attr_reader :translations, :source_paths, :ignore_patterns,
+                :provider, :model, :concurrency, :context_lines,
+                :max_matches_per_key, :output_path, :output_format,
+                :no_cache, :dry_run, :key_filter, :write_back,
+                :swift_functions, :write_back_to_code, :diff_base, :context_prefix,
+                :context_mode, :start_key, :end_key, :include_file_paths,
+                :include_translation_comments, :redact_prompts
+
+    DEFAULT_CONTEXT_PREFIX = 'Context: '
+    DEFAULT_CONTEXT_MODE = 'replace' # "replace" or "append"
+
+    def initialize(**attrs)
+      @translations = attrs[:translations] || []
+      @source_paths = attrs[:source_paths] || ['.']
+      @ignore_patterns = attrs[:ignore_patterns] || []
+      @provider = attrs[:provider] || 'anthropic'
+      @model = attrs[:model]
+      @concurrency = attrs[:concurrency] || 5
+      @context_lines = attrs[:context_lines] || 15
+      @max_matches_per_key = attrs[:max_matches_per_key] || 3
+      @output_path = attrs.key?(:output_path) ? attrs[:output_path] : nil
+      @output_format = attrs[:output_format] || 'csv'
+      @no_cache = attrs.key?(:no_cache) ? attrs[:no_cache] : true
+      @dry_run = attrs[:dry_run] || false
+      @key_filter = attrs[:key_filter]
+      @write_back = attrs[:write_back] || false
+      @write_back_to_code = attrs[:write_back_to_code] || false
+      @swift_functions = attrs[:swift_functions] || default_swift_functions
+      @diff_base = attrs[:diff_base]
+      @context_prefix = attrs.key?(:context_prefix) ? attrs[:context_prefix] : DEFAULT_CONTEXT_PREFIX
+      @context_mode = attrs[:context_mode] || DEFAULT_CONTEXT_MODE
+      @start_key = attrs[:start_key]
+      @end_key = attrs[:end_key]
+      @include_file_paths = attrs.key?(:include_file_paths) ? attrs[:include_file_paths] : false
+      @include_translation_comments = attrs.key?(:include_translation_comments) ? attrs[:include_translation_comments] : true
+      @redact_prompts = attrs.key?(:redact_prompts) ? attrs[:redact_prompts] : true
+    end
+
+    def default_swift_functions
+      %w[NSLocalizedString String(localized: Text(]
+    end
+
+    def self.load(options)
+      if options[:config] && File.exist?(options[:config])
+        from_file(options[:config]).merge_cli(options)
+      else
+        from_cli(options)
+      end
+    end
+
+    def self.from_file(path)
+      yaml = YAML.safe_load_file(path, permitted_classes: []) || {}
+
+      attrs = {
+        translations: parse_translations(yaml['translations']),
+        source_paths: yaml.dig('source', 'paths') || ['.'],
+        ignore_patterns: yaml.dig('source', 'ignore') || default_ignore_patterns,
+        provider: yaml.dig('llm', 'provider') || 'anthropic',
+        model: yaml.dig('llm', 'model'),
+        concurrency: yaml.dig('processing', 'concurrency') || 5,
+        context_lines: yaml.dig('processing', 'context_lines') || 15,
+        max_matches_per_key: yaml.dig('processing', 'max_matches_per_key') || 3,
+        output_path: yaml.dig('output', 'path'),
+        output_format: yaml.dig('output', 'format') || 'csv',
+        write_back: yaml.dig('output', 'write_back') || false,
+        write_back_to_code: yaml.dig('output', 'write_back_to_code') || false,
+        context_mode: yaml.dig('output', 'context_mode'),
+        swift_functions: yaml.dig('swift', 'functions')
+      }
+
+      # Only pass context_prefix when explicitly set in YAML, so initialize default applies
+      prefix = yaml.dig('output', 'context_prefix')
+      attrs[:context_prefix] = prefix unless prefix.nil?
+      attrs[:include_file_paths] = yaml.dig('privacy', 'include_file_paths') unless yaml.dig('privacy', 'include_file_paths').nil?
+      attrs[:include_translation_comments] = yaml.dig('privacy', 'include_translation_comments') unless yaml.dig('privacy', 'include_translation_comments').nil?
+      attrs[:redact_prompts] = yaml.dig('privacy', 'redact_prompts') unless yaml.dig('privacy', 'redact_prompts').nil?
+
+      new(**attrs)
+    end
+
+    def self.from_cli(options)
+      translations = if options[:translations]
+                       options[:translations].split(',').map(&:strip)
+                     else
+                       []
+                     end
+
+      source_paths = if options[:source]
+                       options[:source].split(',').map(&:strip)
+                     else
+                       ['.']
+                     end
+
+      attrs = {
+        translations: translations,
+        source_paths: source_paths,
+        ignore_patterns: default_ignore_patterns,
+        provider: options[:provider] || 'anthropic',
+        model: options[:model],
+        concurrency: options[:concurrency] || 5,
+        context_lines: 15,
+        max_matches_per_key: 3,
+        output_path: options[:output],
+        output_format: options[:format] || 'csv',
+        no_cache: options[:cache].nil? || !options[:cache],
+        dry_run: options[:dry_run] || false,
+        key_filter: options[:keys],
+        write_back: options[:write_back] || false,
+        write_back_to_code: options[:write_back_to_code] || false,
+        diff_base: options[:diff_base],
+        start_key: options[:start_key],
+        end_key: options[:end_key]
+      }
+
+      # Only include if explicitly provided, so Config.new can apply its defaults
+      attrs[:context_prefix] = options[:context_prefix] unless options[:context_prefix].nil?
+      attrs[:context_mode] = options[:context_mode] if options[:context_mode]
+      attrs[:include_file_paths] = options[:include_file_paths] unless options[:include_file_paths].nil?
+      attrs[:include_translation_comments] = options[:include_translation_comments] unless options[:include_translation_comments].nil?
+      attrs[:redact_prompts] = options[:redact_prompts] unless options[:redact_prompts].nil?
+
+      new(**attrs)
+    end
+
+    # Merge CLI options over config-file values.
+    # Only options explicitly passed by the user (non-nil) are merged.
+    # Thor options without defaults are nil when not passed, so this
+    # correctly preserves config-file values for unspecified flags.
+    def merge_cli(options)
+      @translations = options[:translations].split(',').map(&:strip) if options[:translations]
+      @source_paths = options[:source].split(',').map(&:strip) if options[:source]
+      merge_cli_scalar_options(options)
+      merge_cli_boolean_options(options)
+      self
+    end
+
+    def self.parse_translations(translations)
+      return [] unless translations
+
+      translations.map do |t|
+        t.is_a?(Hash) ? t['path'] : t
+      end
+    end
+
+    def self.default_ignore_patterns
+      [
+        '**/node_modules/**',
+        '**/vendor/**',
+        '**/.git/**',
+        '**/build/**',
+        '**/dist/**',
+        '**/*.min.js',
+        '**/*.test.*',
+        '**/*.spec.*',
+        '**/Pods/**',
+        '**/Carthage/**',
+        '**/.build/**',
+        '**/DerivedData/**',
+        '**/*Tests.swift',
+        '**/*Tests.kt',
+        '**/*Test.java',
+        '**/*Test.kt'
+      ]
+    end
+
+    private
+
+    def merge_cli_scalar_options(options)
+      scalar_mappings = {
+        key_filter: :keys,
+        output_path: :output,
+        output_format: :format,
+        provider: :provider,
+        model: :model,
+        concurrency: :concurrency,
+        diff_base: :diff_base,
+        context_prefix: :context_prefix,
+        context_mode: :context_mode,
+        start_key: :start_key,
+        end_key: :end_key
+      }
+
+      scalar_mappings.each do |attr_name, option_name|
+        value = options[option_name]
+        instance_variable_set(:"@#{attr_name}", value) unless value.nil?
+      end
+    end
+
+    def merge_cli_boolean_options(options)
+      boolean_mappings = {
+        dry_run: :dry_run,
+        write_back: :write_back,
+        write_back_to_code: :write_back_to_code,
+        include_file_paths: :include_file_paths,
+        include_translation_comments: :include_translation_comments,
+        redact_prompts: :redact_prompts
+      }
+
+      @no_cache = !options[:cache] unless options[:cache].nil?
+      boolean_mappings.each do |attr_name, option_name|
+        value = options[option_name]
+        instance_variable_set(:"@#{attr_name}", value) unless value.nil?
+      end
+    end
+  end
+end