i18n-tasks 1.0.15 → 1.1.0

data/lib/i18n/tasks/scanners/ruby_scanner.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "i18n/tasks/logging"
+require "i18n/tasks/scanners/file_scanner"
+require "i18n/tasks/scanners/relative_keys"
+require "i18n/tasks/scanners/ruby_ast_call_finder"
+require "i18n/tasks/scanners/ruby_parser_factory"
+require "i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher"
+require "i18n/tasks/scanners/ast_matchers/message_receivers_matcher"
+require "i18n/tasks/scanners/ast_matchers/rails_model_matcher"
+require "i18n/tasks/scanners/prism_scanners/visitor"
+
+module I18n::Tasks::Scanners
+  # Scan for I18n.translate calls using whitequark/parser primarily and Prism if configured.
+  class RubyScanner < FileScanner
+    MAGIC_COMMENT_SKIP_PRISM = "i18n-tasks-skip-prism"
+    include RelativeKeys
+    include AST::Sexp
+    include ::I18n::Tasks::Logging
+
+    MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/
+
+    protected
+
+    # Extract all occurrences of translate calls from the file at the given path.
+    #
+    # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
+    def scan_file(path)
+      if config[:prism]
+        prism_parse_file(path)
+      else
+        ast_parser_parse_file(path)
+      end
+    rescue => e
+      raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
+    end
+
+    def ast_parser_parse_file(path)
+      setup_ast_parser
+      ast, comments = path_to_ast_and_comments(path)
+
+      ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
+    end
+
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      @parser.reset
+      @parser.parse_with_comments(make_buffer(path))
+    end
+
+    # Create an {Parser::Source::Buffer} with the given contents.
+    # The contents are assigned a {Parser::Source::Buffer#raw_source}.
+    #
+    # @param path [String] Path to assign as the buffer name.
+    # @param contents [String]
+    # @return [Parser::Source::Buffer] file contents
+    def make_buffer(path, contents = read_file(path))
+      Parser::Source::Buffer.new(path).tap do |buffer|
+        buffer.raw_source = contents
+      end
+    end
+
+    # Convert an array of {Parser::Source::Comment} to occurrences.
+    #
+    # @param path Path to file
+    # @param ast Parser::AST::Node
+    # @param comments [Parser::Source::Comment]
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def comments_to_occurences(path, ast, comments)
+      magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
+      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
+        h.transform_values!(&:first)
+      end.invert
+
+      magic_comments.flat_map do |comment|
+        @parser.reset
+        associated_node = comment_to_node[comment]
+        ast = @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, "").split(/\s+(?=t)/).join("; ")))
+        calls = RubyAstCallFinder.new.collect_calls(ast)
+        results = []
+
+        # method_name is not available at this stage
+        calls.each do |(send_node, _method_name)|
+          @matchers.each do |matcher|
+            result = matcher.convert_to_key_occurrences(
+              send_node,
+              nil,
+              location: associated_node || comment.location
+            )
+            next unless result
+
+            if result.is_a?(Array) && result.first.is_a?(Array)
+              results.concat(result)
+            else
+              results << result
+            end
+          end
+        end
+
+        results
+      end
+    end
+
+    # Convert {Parser::AST::Node} to occurrences.
+    #
+    # @param ast {Parser::Source::Comment}
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def ast_to_occurences(ast)
+      calls = RubyAstCallFinder.new.collect_calls(ast)
+      results = []
+      calls.each do |send_node, method_name|
+        @matchers.each do |matcher|
+          result = matcher.convert_to_key_occurrences(send_node, method_name)
+          next unless result
+
+          if result.is_a?(Array) && result.first.is_a?(Array)
+            results.concat(result)
+          else
+            results << result
+          end
+        end
+      end
+
+      results
+    end
+
+    def setup_ast_parser
+      @parser ||= RubyParserFactory.create_parser
+      @magic_comment_parser ||= RubyParserFactory.create_parser
+      setup_ast_matchers
+    end
+
+    def setup_ast_matchers
+      return if defined?(@matchers)
+
+      if config[:receiver_messages]
+        @matchers = config[:receiver_messages].map do |receiver, message|
+          AstMatchers::MessageReceiversMatcher.new(
+            receivers: [receiver],
+            message: message,
+            scanner: self
+          )
+        end
+      else
+        @matchers = %i[t t! translate translate!].map do |message|
+          AstMatchers::MessageReceiversMatcher.new(
+            receivers: [
+              AST::Node.new(:const, [nil, :I18n]),
+              nil
+            ],
+            message: message,
+            scanner: self
+          )
+        end
+
+        Array(config[:ast_matchers]).each do |class_name|
+          @matchers << ActiveSupport::Inflector.constantize(class_name).new(scanner: self)
+        end
+      end
+    end
+
+    # ---------- Prism parser below ----------
+
+    # Extract all occurrences of translate calls from the file at the given path.
+    # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
+    def prism_parse_file(path)
+      process_prism_results(path, Prism.parse_file(path))
+    end
+
+    # This method handles only parsing to be able to test it properly.
+    # Therefore it cannot handle the parsing itself.
+    def process_prism_results(path, parse_results)
+      comments = parse_results.attach_comments!
+      parsed = parse_results.value
+
+      # Check for magic comment to skip prism parsing, fallback to Parser AST
+      return ast_parser_parse_file(path) if skip_prism_comment?(comments)
+
+      visitor = I18n::Tasks::Scanners::PrismScanners::Visitor.new(
+        rails: config[:prism] != "ruby",
+        file_path: path
+      )
+      parsed.accept(visitor)
+
+      occurrences = []
+      visitor.process.each do |translation_call|
+        result = translation_call.occurrences(path)
+        next unless result
+
+        if result.is_a?(Array) && result.first.is_a?(Array)
+          occurrences.concat(result)
+        else
+          occurrences << result
+        end
+      end
+
+      occurrences
+    end
+
+    def skip_prism_comment?(comments)
+      comments.any? do |comment|
+        content =
+          comment.respond_to?(:slice) ? comment.slice : comment.location.slice
+        content.include?(MAGIC_COMMENT_SKIP_PRISM)
+      end
+    end
+  end
+
+  class RubyAstScanner < RubyScanner
+    def initialize(**args)
+      warn_deprecated("RubyAstScanner is deprecated, use RubyScanner instead")
+      super
+    end
+  end
+
+  class PrismScanner < RubyScanner
+    def initialize(**args)
+      warn_deprecated('PrismScanner is deprecated, use RubyScanner with prism: "rails" or prism: "ruby" instead')
+      super
+    end
+  end
+end

data/lib/i18n/tasks/scanners/scanner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/scanners/results/key_occurrences'
+require "i18n/tasks/scanners/results/key_occurrences"
 
 module I18n::Tasks::Scanners
   # Describes the API of a scanner.
@@ -11,7 +11,7 @@ module I18n::Tasks::Scanners
     # @abstract
     # @return [Array<Results::KeyOccurrences>] the keys found by this scanner and their occurrences.
     def keys
-      fail 'Unimplemented'
+      fail "Unimplemented"
     end
   end
 end

data/lib/i18n/tasks/scanners/scanner_multiplexer.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/scanners/scanner'
+require "i18n/tasks/scanners/scanner"
 
 module I18n::Tasks::Scanners
   # Run multiple {Scanner Scanners} and merge their results.

data/lib/i18n/tasks/split_key.rb

@@ -21,7 +21,7 @@ module I18n
 
         parts = []
         current_parenthesis_end_char = nil
-        part = ''
+        part = ""
         key.each_char.with_index do |char, index|
           if current_parenthesis_end_char
             part += char
@@ -29,14 +29,14 @@ module I18n
           elsif START_KEYS.include?(char)
             part += char
             current_parenthesis_end_char = END_KEYS[char]
-          elsif char == '.'
+          elsif char == "."
             parts << part
             if parts.size + 1 == max
               remaining = key[(index + 1)..]
               parts << remaining unless remaining.empty?
               return parts
             end
-            part = ''
+            part = ""
           else
             part += char
           end
@@ -44,7 +44,7 @@ module I18n
 
         return parts if part.empty?
 
-        current_parenthesis_end_char ? parts.concat(part.split('.')) : parts << part
+        current_parenthesis_end_char ? parts.concat(part.split(".")) : parts << part
       end
 
       def last_key_part(key)

data/lib/i18n/tasks/stats.rb

@@ -6,15 +6,15 @@ module I18n::Tasks
       key_count = forest.leaves.count
       locale_count = forest.count
       if key_count.zero?
-        { key_count: 0 }
+        {key_count: 0}
       else
         {
-          locales: forest.map(&:key).join(', '),
+          locales: forest.map(&:key).join(", "),
           key_count: key_count,
           locale_count: locale_count,
           per_locale_avg: forest.inject(0) { |sum, f| sum + f.leaves.count } / locale_count,
           key_segments_avg: format(
-            '%.1f', forest.leaves.inject(0) { |sum, node| sum + node.walk_to_root.count - 1 } / key_count.to_f
+            "%.1f", forest.leaves.inject(0) { |sum, node| sum + node.walk_to_root.count - 1 } / key_count.to_f
           ),
           value_chars_avg: forest.leaves.inject(0) { |sum, node| sum + node.value.to_s.length } / key_count
         }

data/lib/i18n/tasks/translation.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/deepl_translator'
-require 'i18n/tasks/translators/google_translator'
-require 'i18n/tasks/translators/openai_translator'
-require 'i18n/tasks/translators/watsonx_translator'
-require 'i18n/tasks/translators/yandex_translator'
+require "i18n/tasks/translators/deepl_translator"
+require "i18n/tasks/translators/google_translator"
+require "i18n/tasks/translators/openai_translator"
+require "i18n/tasks/translators/watsonx_translator"
+require "i18n/tasks/translators/yandex_translator"
 
 module I18n::Tasks
   module Translation

data/lib/i18n/tasks/translators/base_translator.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
+require "i18n/tasks/data/language_names"
+
 module I18n::Tasks
   module Translators
     class BaseTranslator
       include ::I18n::Tasks::Logging
+
       # @param [I18n::Tasks::BaseTask] i18n_tasks
       def initialize(i18n_tasks)
         @i18n_tasks = i18n_tasks
@@ -16,9 +19,24 @@ module I18n::Tasks
         forest.inject @i18n_tasks.empty_forest do |result, root|
           pairs = root.key_values(root: true)
 
-          @progress_bar = ProgressBar.create(total: pairs.flatten.size, format: '%a <%B> %e %c/%C (%p%%)')
+          @progress_bar = ProgressBar.create(total: pairs.flatten.size, format: "%a <%B> %e %c/%C (%p%%)")
+
+          begin
+            translated = translate_pairs(pairs, to: root.key, from: from)
+          rescue => e
+            warn "Translation for locale #{root.key} failed: #{e.message}"
+            # If translate_pairs raised, try to salvage any partial translations
+            # by attempting to translate each slice individually and collecting successes.
+            translated = []
+            pairs.group_by { |k_v| @i18n_tasks.html_key? k_v[0], from }.each do |_is_html, list_slice|
+              translated.concat(fetch_translations(list_slice, to: root.key, from: from))
+            rescue => e2
+              warn "Partial translation failed for locale #{root.key}: #{e2.message} - leaving keys untranslated"
+              # leave the original list_slice untranslated
+              translated.concat(list_slice)
+            end
+          end
 
-          translated = translate_pairs(pairs, to: root.key, from: from)
           result.merge! Data::Tree::Siblings.from_flat_pairs(translated)
         end
       end
@@ -37,6 +55,10 @@ module I18n::Tasks
         list -= reference_key_vals
         result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
           fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
+        rescue => e
+          warn "Translation slice failed: #{e.message} - leaving slice untranslated"
+          # Return the original untranslated slice so already completed translations are preserved
+          list_slice
         end.reduce(:+) || []
         result.concat(reference_key_vals)
         result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
@@ -108,8 +130,7 @@ module I18n::Tasks
         end
       end
 
-      INTERPOLATION_KEY_RE = /%\{[^}]+}/.freeze
-      UNTRANSLATABLE_STRING = 'X__'
+      INTERPOLATION_KEY_RE = /%\{[^}]+}/
 
       # @param [String] value
       # @return [String] 'hello, %{name}' => 'hello, <round-trippable string>'
@@ -117,7 +138,7 @@ module I18n::Tasks
         i = -1
         value.gsub INTERPOLATION_KEY_RE do
           i += 1
-          "#{UNTRANSLATABLE_STRING}#{i}"
+          "X__#{i}"
         end
       end
 
@@ -125,13 +146,13 @@ module I18n::Tasks
       # @param [String] translated
       # @return [String] 'hello, <round-trippable string>' => 'hello, %{name}'
       def restore_interpolations(untranslated, translated)
-        return translated if untranslated !~ INTERPOLATION_KEY_RE
+        return translated if !INTERPOLATION_KEY_RE.match?(untranslated)
 
         values = untranslated.scan(INTERPOLATION_KEY_RE)
-        translated.gsub(/#{Regexp.escape(UNTRANSLATABLE_STRING)}\d+/i) do |m|
-          values[m[UNTRANSLATABLE_STRING.length..].to_i]
+        translated.gsub(/X__(\d+)/) do |m|
+          values[$1.to_i]
         end
-      rescue StandardError => e
+      rescue => e
         raise_interpolation_error(untranslated, translated, e)
       end
 
@@ -148,24 +169,29 @@ module I18n::Tasks
       # @param [Hash] options
       # @return [Array<String>]
       # @abstract
-      def translate_values(list, **options); end
+      def translate_values(list, **options)
+      end
 
       # @param [Hash] options
       # @return [Hash]
       # @abstract
-      def options_for_translate_values(options); end
+      def options_for_translate_values(options)
+      end
 
       # @return [Hash]
       # @abstract
-      def options_for_html; end
+      def options_for_html
+      end
 
       # @return [Hash]
       # @abstract
-      def options_for_plain; end
+      def options_for_plain
+      end
 
       # @return [String]
       # @abstract
-      def no_results_error_message; end
+      def no_results_error_message
+      end
     end
   end
 end

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
+require "i18n/tasks/translators/base_translator"
 
 module I18n::Tasks::Translators
   class DeeplTranslator < BaseTranslator
@@ -11,7 +11,7 @@ module I18n::Tasks::Translators
 
     def initialize(*)
       begin
-        require 'deepl'
+        require "deepl"
       rescue LoadError
         raise ::I18n::Tasks::CommandError, "Add gem 'deepl-rb' to your Gemfile to use this command"
       end
@@ -44,15 +44,15 @@ module I18n::Tasks::Translators
     def options_for_translate_values(**options)
       extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
 
-      extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
+      extra_options.merge({ignore_tags: %w[i18n]}).merge(options)
     end
 
     def options_for_html
-      { tag_handling: 'xml' }
+      {tag_handling: "xml"}
     end
 
     def options_for_plain
-      { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
+      {preserve_formatting: true, tag_handling: "xml", html_escape: true}
     end
 
     # @param [String] value
@@ -67,20 +67,20 @@ module I18n::Tasks::Translators
     def restore_interpolations(untranslated, translated)
       return translated if untranslated !~ INTERPOLATION_KEY_RE
 
-      translated.gsub(%r{</?i18n>}, '')
-    rescue StandardError => e
+      translated.gsub(%r{</?i18n>}, "")
+    rescue => e
       raise_interpolation_error(untranslated, translated, e)
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.deepl_translate.errors.no_results')
+      I18n.t("i18n_tasks.deepl_translate.errors.no_results")
     end
 
     private
 
     # Convert 'es-ES' to 'ES', en-us to EN
     def to_deepl_source_locale(locale)
-      locale.to_s.split('-', 2).first.upcase
+      locale.to_s.split("-", 2).first.upcase
     end
 
     # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
@@ -88,10 +88,10 @@ module I18n::Tasks::Translators
       locale_aliases = @i18n_tasks.translation_config[:deepl_locale_aliases]
       locale = locale_aliases[locale.to_s.downcase] || locale if locale_aliases.is_a?(Hash)
 
-      loc, sub = locale.to_s.split('-')
+      loc, sub = locale.to_s.split("-")
       if SPECIFIC_TARGETS.include?(loc)
         # Must see how the deepl api evolves, so this could be an error in the future
-        warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
+        warn_deprecated I18n.t("i18n_tasks.deepl_translate.errors.specific_target_missing") unless sub
         locale.to_s.upcase
       else
         loc.upcase
@@ -102,7 +102,7 @@ module I18n::Tasks::Translators
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]
       version = @i18n_tasks.translation_config[:deepl_version]
-      fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.no_api_key') if api_key.blank?
+      fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.deepl_translate.errors.no_api_key") if api_key.blank?
 
       DeepL.configure do |config|
         config.auth_key = api_key
@@ -111,9 +111,12 @@ module I18n::Tasks::Translators
       end
     end
 
+    # The Free API endpoint doesn’t expose glossaries via DeepL.glossaries.list,
+    # so if no API-backed glossary is found, fall back to the first ID from i18n-config.yml.
     def options_with_glossary(options, from, to)
-      glossary = find_glossary(from, to)
-      glossary ? { glossary_id: glossary.id }.merge(options) : options
+      configured = @i18n_tasks.translation_config[:deepl_glossary_ids]
+      gid = find_glossary(from, to)&.id || configured&.first
+      gid ? {glossary_id: gid}.merge(options) : options
     end
 
     def all_ready_glossaries