i18n-tasks 0.9.37 → 1.0.13

data/lib/i18n/tasks/scanners/ruby_ast_call_finder.rb

@@ -1,56 +1,31 @@
 # frozen_string_literal: true
 
 require 'ast'
-require 'set'
+
 module I18n::Tasks::Scanners
   class RubyAstCallFinder
     include AST::Processor::Mixin
 
-    # @param receiver_messages [Set<Pair<[nil, AST::Node>, Symbol>>] The receiver-message pairs to look for.
-    def initialize(receiver_messages:)
-      super()
-      @message_receivers = receiver_messages.each_with_object({}) do |(receiver, message), t|
-        (t[message] ||= []) << receiver
-      end
-    end
-
     # @param root_node [Parser::AST:Node]
-    # @yieldparam send_node [Parser::AST:Node]
-    # @yieldparam method_name [nil, String] the surrounding method's name.
-    def find_calls(root_node, &block)
-      @callback = block
-      process root_node
-    ensure
-      @callback = nil
-    end
-
-    # @param root_node (see #find_calls)
-    # @yieldparam (see #find_calls)
-    # @return [Array<block return values excluding nils>]
+    # @return [Pair<Parser::AST::Node, method_name as string>] for all nodes with :send type
     def collect_calls(root_node)
-      results = []
-      find_calls root_node do |send_node, method_name|
-        result = yield send_node, method_name
-        results << result if result
-      end
-      results
+      @results = []
+      process(root_node)
+      @results
     end
 
     def on_def(node)
       @method_name = node.children[0]
-      handler_missing node
+      handler_missing(node)
     ensure
       @method_name = nil
     end
 
     def on_send(send_node)
-      receiver = send_node.children[0]
-      message  = send_node.children[1]
-      valid_receivers = @message_receivers[message]
-      # use `any?` because `include?` checks type equality, but the receiver is a Parser::AST::Node != AST::Node.
-      @callback.call(send_node, @method_name) if valid_receivers&.any? { |r| r == receiver }
+      @results << [send_node, @method_name]
+
       # always invoke handler_missing to get nested translations in children
-      handler_missing send_node
+      handler_missing(send_node)
       nil
     end
 

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -3,27 +3,23 @@
 require 'i18n/tasks/scanners/file_scanner'
 require 'i18n/tasks/scanners/relative_keys'
 require 'i18n/tasks/scanners/ruby_ast_call_finder'
+require 'i18n/tasks/scanners/ast_matchers/message_receivers_matcher'
+require 'i18n/tasks/scanners/ast_matchers/rails_model_matcher'
 require 'parser/current'
 
-# rubocop:disable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity
-# TODO: make this class more readable.
-
 module I18n::Tasks::Scanners
   # Scan for I18n.translate calls using whitequark/parser
-  class RubyAstScanner < FileScanner # rubocop:disable Metrics/ClassLength
+  class RubyAstScanner < FileScanner
     include RelativeKeys
     include AST::Sexp
 
     MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/.freeze
-    RECEIVER_MESSAGES = [nil, AST::Node.new(:const, [nil, :I18n])].product(%i[t t! translate translate!])
 
     def initialize(**args)
       super(**args)
       @parser = ::Parser::CurrentRuby.new
       @magic_comment_parser = ::Parser::CurrentRuby.new
-      @call_finder = RubyAstCallFinder.new(
-        receiver_messages: config[:receiver_messages] || RECEIVER_MESSAGES
-      )
+      @matchers = setup_matchers
     end
 
     protected
@@ -32,178 +28,117 @@ module I18n::Tasks::Scanners
     #
     # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
     def scan_file(path)
-      @parser.reset
-      ast, comments = @parser.parse_with_comments(make_buffer(path))
-
-      results = @call_finder.collect_calls ast do |send_node, method_name|
-        send_node_to_key_occurrence(send_node, method_name)
-      end
+      ast, comments = path_to_ast_and_comments(path)
 
-      magic_comments  = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
-      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
-        # transform_values is only available in ActiveSupport 4.2+
-        h.each { |k, v| h[k] = v.first }
-      end.invert
-      results + (magic_comments.flat_map do |comment|
-        @parser.reset
-        associated_node = comment_to_node[comment]
-        @call_finder.collect_calls(
-          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
-        ) do |send_node, _method_name|
-          # method_name is not available at this stage
-          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
-        end
-      end)
+      ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
     rescue Exception => e # rubocop:disable Lint/RescueException
       raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
     end
 
-    # @param send_node [Parser::AST::Node]
-    # @param method_name [Symbol, nil]
-    # @param location [Parser::Source::Map]
-    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
-    def send_node_to_key_occurrence(send_node, method_name, location: send_node.loc)
-      if (first_arg_node = send_node.children[2]) &&
-         (key = extract_string(first_arg_node))
-        if (second_arg_node = send_node.children[3]) &&
-           second_arg_node.type == :hash
-          if (scope_node = extract_hash_pair(second_arg_node, 'scope'))
-            scope = extract_string(scope_node.children[1],
-                                   array_join_with: '.', array_flatten: true, array_reject_blank: true)
-            return nil if scope.nil? && scope_node.type != :nil
-
-            key = [scope, key].join('.') unless scope == ''
-          end
-          default_arg = if (default_arg_node = extract_hash_pair(second_arg_node, 'default'))
-                          extract_string(default_arg_node.children[1])
-                        end
-        end
-        full_key = if send_node.children[0].nil?
-                     # Relative keys only work if called via `t()` but not `I18n.t()`:
-                     absolute_key(key, location.expression.source_buffer.name, calling_method: method_name)
-                   else
-                     key
-                   end
-        [full_key, range_to_occurrence(key, location.expression, default_arg: default_arg)]
-      end
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      @parser.reset
+      @parser.parse_with_comments(make_buffer(path))
     end
 
-    # Extract a hash pair with a given literal key.
-    #
-    # @param node [AST::Node] a node of type `:hash`.
-    # @param key [String] node key as a string (indifferent symbol-string matching).
-    # @return [AST::Node, nil] a node of type `:pair` or nil.
-    def extract_hash_pair(node, key)
-      node.children.detect do |child|
-        next unless child.type == :pair
-
-        key_node = child.children[0]
-        %i[sym str].include?(key_node.type) && key_node.children[0].to_s == key
-      end
+    def keys_relative_to_calling_method?(path)
+      /controllers|mailers/.match(path)
     end
 
-    # If the node type is of `%i(sym str int false true)`, return the value as a string.
-    # Otherwise, if `config[:strict]` is `false` and the type is of `%i(dstr dsym)`,
-    # return the source as if it were a string.
+    # Create an {Parser::Source::Buffer} with the given contents.
+    # The contents are assigned a {Parser::Source::Buffer#raw_source}.
     #
-    # @param node [Parser::AST::Node]
-    # @param array_join_with [String, nil] if set to a string, arrays will be processed and their elements joined.
-    # @param array_flatten [Boolean] if true, nested arrays are flattened,
-    #     otherwise their source is copied and surrounded by #{}. No effect unless `array_join_with` is set.
-    # @param array_reject_blank [Boolean] if true, empty strings and `nil`s are skipped.
-    #      No effect unless `array_join_with` is set.
-    # @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode
-    #     or the node type is not supported.
-    def extract_string(node, array_join_with: nil, array_flatten: false, array_reject_blank: false)
-      if %i[sym str int].include?(node.type)
-        node.children[0].to_s
-      elsif %i[true false].include?(node.type)
-        node.type.to_s
-      elsif node.type == :nil
-        ''
-      elsif node.type == :array && array_join_with
-        extract_array_as_string(
-          node,
-          array_join_with: array_join_with,
-          array_flatten: array_flatten,
-          array_reject_blank: array_reject_blank
-        ).tap do |str|
-          # `nil` is returned when a dynamic value is encountered in strict mode. Propagate:
-          return nil if str.nil?
-        end
-      elsif !config[:strict] && %i[dsym dstr].include?(node.type)
-        node.children.map do |child|
-          if %i[sym str].include?(child.type)
-            child.children[0].to_s
-          else
-            child.loc.expression.source
-          end
-        end.join
+    # @param path [String] Path to assign as the buffer name.
+    # @param contents [String]
+    # @return [Parser::Source::Buffer] file contents
+    def make_buffer(path, contents = read_file(path))
+      Parser::Source::Buffer.new(path).tap do |buffer|
+        buffer.raw_source = contents
       end
     end
 
-    # Extract an array as a single string.
+    # Convert an array of {Parser::Source::Comment} to occurrences.
     #
-    # @param array_join_with [String] joiner of the array elements.
-    # @param array_flatten [Boolean] if true, nested arrays are flattened,
-    #     otherwise their source is copied and surrounded by #{}.
-    # @param array_reject_blank [Boolean] if true, empty strings and `nil`s are skipped.
-    # @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode.
-    def extract_array_as_string(node, array_join_with:, array_flatten: false, array_reject_blank: false)
-      children_strings = node.children.map do |child|
-        if %i[sym str int true false].include?(child.type)
-          extract_string child
-        else
-          # ignore dynamic argument in strict mode
-          return nil if config[:strict]
-
-          if %i[dsym dstr].include?(child.type) || (child.type == :array && array_flatten)
-            extract_string(child, array_join_with: array_join_with)
-          else
-            "\#{#{child.loc.expression.source}}"
+    # @param path Path to file
+    # @param ast Parser::AST::Node
+    # @param comments [Parser::Source::Comment]
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def comments_to_occurences(path, ast, comments)
+      magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
+      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
+        h.transform_values!(&:first)
+      end.invert
+
+      magic_comments.flat_map do |comment|
+        @parser.reset
+        associated_node = comment_to_node[comment]
+        ast = @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
+        calls = RubyAstCallFinder.new.collect_calls(ast)
+        results = []
+
+        # method_name is not available at this stage
+        calls.each do |send_node, _method_name|
+          @matchers.each do |matcher|
+            result = matcher.convert_to_key_occurrences(
+              send_node,
+              nil,
+              location: associated_node || comment.location
+            )
+            results << result if result
           end
         end
+
+        results
       end
-      if array_reject_blank
-        children_strings.reject! do |x|
-          # empty strings and nils in the scope argument are ignored by i18n
-          x == ''
+    end
+
+    # Convert {Parser::AST::Node} to occurrences.
+    #
+    # @param ast {Parser::Source::Comment}
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def ast_to_occurences(ast)
+      calls = RubyAstCallFinder.new.collect_calls(ast)
+      results = []
+      calls.each do |send_node, method_name|
+        @matchers.each do |matcher|
+          result = matcher.convert_to_key_occurrences(send_node, method_name)
+          results << result if result
         end
       end
-      children_strings.join(array_join_with)
-    end
 
-    def keys_relative_to_calling_method?(path)
-      /controllers|mailers/.match(path)
+      results
     end
 
-    # @param raw_key [String]
-    # @param range [Parser::Source::Range]
-    # @param default_arg [String, nil]
-    # @return [Results::Occurrence]
-    def range_to_occurrence(raw_key, range, default_arg: nil)
-      Results::Occurrence.new(
-        path: range.source_buffer.name,
-        pos: range.begin_pos,
-        line_num: range.line,
-        line_pos: range.column,
-        line: range.source_line,
-        raw_key: raw_key,
-        default_arg: default_arg
-      )
-    end
+    def setup_matchers
+      if config[:receiver_messages]
+        config[:receiver_messages].map do |receiver, message|
+          AstMatchers::MessageReceiversMatcher.new(
+            receivers: [receiver],
+            message: message,
+            scanner: self
+          )
+        end
+      else
+        matchers = %i[t t! translate translate!].map do |message|
+          AstMatchers::MessageReceiversMatcher.new(
+            receivers: [
+              AST::Node.new(:const, [nil, :I18n]),
+              nil
+            ],
+            message: message,
+            scanner: self
+          )
+        end
 
-    # Create an {Parser::Source::Buffer} with the given contents.
-    # The contents are assigned a {Parser::Source::Buffer#raw_source}.
-    #
-    # @param path [String] Path to assign as the buffer name.
-    # @param contents [String]
-    # @return [Parser::Source::Buffer] file contents
-    def make_buffer(path, contents = read_file(path))
-      Parser::Source::Buffer.new(path).tap do |buffer|
-        buffer.raw_source = contents
+        Array(config[:ast_matchers]).each do |class_name|
+          matchers << ActiveSupport::Inflector.constantize(class_name).new(scanner: self)
+        end
+
+        matchers
       end
     end
   end
 end
-# rubocop:enable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity

data/lib/i18n/tasks/scanners/ruby_key_literals.rb

@@ -15,12 +15,12 @@ module I18n::Tasks::Scanners
     # @param literal [String] e.g: "key", 'key', or :key.
     # @return [String] key
     def strip_literal(literal)
-      literal = literal[1..-1] if literal[0] == ':'
+      literal = literal[1..] if literal[0] == ':'
       literal = literal[1..-2] if literal[0] == "'" || literal[0] == '"'
       literal
     end
 
-    VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!:;À-ž\/])/.freeze
+    VALID_KEY_CHARS = %r{(?:[[:word:]]|[-.?!:;À-ž/])}.freeze
     VALID_KEY_RE    = /^#{VALID_KEY_CHARS}+$/.freeze
 
     def valid_key?(key)

data/lib/i18n/tasks/split_key.rb

@@ -20,7 +20,7 @@ module I18n
           parts << part
           pos += part.length + 1
           if parts.length + 1 >= max
-            parts << key[pos..-1] unless pos == key.length
+            parts << key[pos..] unless pos == key.length
             break
           end
         end

data/lib/i18n/tasks/translation.rb

@@ -2,13 +2,14 @@
 
 require 'i18n/tasks/translators/deepl_translator'
 require 'i18n/tasks/translators/google_translator'
+require 'i18n/tasks/translators/openai_translator'
 require 'i18n/tasks/translators/yandex_translator'
 
 module I18n::Tasks
   module Translation
     # @param [I18n::Tasks::Tree::Siblings] forest to translate to the locales of its root nodes
     # @param [String] from locale
-    # @param [:deepl, :google, :yandex] backend
+    # @param [:deepl, :openai, :google, :yandex] backend
     # @return [I18n::Tasks::Tree::Siblings] translated forest
     def translate_forest(forest, from:, backend: :google)
       case backend
@@ -16,6 +17,8 @@ module I18n::Tasks
         Translators::DeeplTranslator.new(self).translate_forest(forest, from)
       when :google
         Translators::GoogleTranslator.new(self).translate_forest(forest, from)
+      when :openai
+        Translators::OpenAiTranslator.new(self).translate_forest(forest, from)
       when :yandex
         Translators::YandexTranslator.new(self).translate_forest(forest, from)
       else

data/lib/i18n/tasks/translators/base_translator.rb

@@ -3,6 +3,7 @@
 module I18n::Tasks
   module Translators
     class BaseTranslator
+      include ::I18n::Tasks::Logging
       # @param [I18n::Tasks::BaseTask] i18n_tasks
       def initialize(i18n_tasks)
         @i18n_tasks = i18n_tasks
@@ -31,7 +32,7 @@ module I18n::Tasks
         reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
         list -= reference_key_vals
         result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
-          fetch_translations list_slice, opts.merge(is_html ? options_for_html : options_for_plain)
+          fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
         end.reduce(:+) || []
         result.concat(reference_key_vals)
         result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
@@ -41,34 +42,36 @@ module I18n::Tasks
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<[String, Object]>] translated list
       def fetch_translations(list, opts)
-        from_values(list, translate_values(to_values(list), **options_for_translate_values(**opts))).tap do |result|
+        options = options_for_translate_values(**opts)
+        from_values(list, translate_values(to_values(list, options), **options), options).tap do |result|
           fail CommandError, no_results_error_message if result.blank?
         end
       end
 
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<String>] values for translation extracted from list
-      def to_values(list)
-        list.map { |l| dump_value l[1] }.flatten.compact
+      def to_values(list, opts)
+        list.map { |l| dump_value(l[1], opts) }.flatten.compact
       end
 
       # @param [Array<[String, Object]>] list
       # @param [Array<String>] translated_values
       # @return [Array<[String, Object]>] translated key-value pairs
-      def from_values(list, translated_values)
+      def from_values(list, translated_values, opts)
         keys = list.map(&:first)
         untranslated_values = list.map(&:last)
-        keys.zip parse_value(untranslated_values, translated_values.to_enum)
+        keys.zip parse_value(untranslated_values, translated_values.to_enum, opts)
       end
 
       # Prepare value for translation.
       # @return [String, Array<String, nil>, nil] value for Google Translate or nil for non-string values
-      def dump_value(value)
+      def dump_value(value, opts)
         case value
         when Array
           # dump recursively
-          value.map { |v| dump_value v }
+          value.map { |v| dump_value(v, opts) }
         when String
+          value = CGI.escapeHTML(value) if opts[:html_escape]
           replace_interpolations value unless value.empty?
         end
       end
@@ -77,16 +80,18 @@ module I18n::Tasks
       # @param [Object] untranslated
       # @param [Enumerator] each_translated
       # @return [Object] final translated value
-      def parse_value(untranslated, each_translated)
+      def parse_value(untranslated, each_translated, opts)
         case untranslated
         when Array
           # implode array
-          untranslated.map { |from| parse_value(from, each_translated) }
+          untranslated.map { |from| parse_value(from, each_translated, opts) }
         when String
           if untranslated.empty?
             untranslated
           else
-            restore_interpolations untranslated, each_translated.next
+            value = each_translated.next
+            value = CGI.unescapeHTML(value) if opts[:html_escape]
+            restore_interpolations(untranslated, value)
           end
         else
           untranslated
@@ -114,7 +119,7 @@ module I18n::Tasks
 
         values = untranslated.scan(INTERPOLATION_KEY_RE)
         translated.gsub(/#{Regexp.escape(UNTRANSLATABLE_STRING)}\d+/i) do |m|
-          values[m[UNTRANSLATABLE_STRING.length..-1].to_i]
+          values[m[UNTRANSLATABLE_STRING.length..].to_i]
         end
       rescue StandardError => e
         raise_interpolation_error(untranslated, translated, e)

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -4,6 +4,11 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class DeeplTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    # those languages must be specified with their sub-kind e.g en-us
+    SPECIFIC_TARGETS = %w[en pt].freeze
+
     def initialize(*)
       begin
         require 'deepl'
@@ -17,16 +22,22 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, from:, to:, **options)
-      result = DeepL.translate(list, to_deepl_compatible_locale(from), to_deepl_compatible_locale(to), options)
-      if result.is_a?(DeepL::Resources::Text)
-        [result.text]
-      else
-        result.map(&:text)
+      results = []
+      list.each_slice(BATCH_SIZE) do |parts|
+        res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options)
+        if res.is_a?(DeepL::Resources::Text)
+          results << res.text
+        else
+          results += res.map(&:text)
+        end
       end
+      results
     end
 
     def options_for_translate_values(**options)
-      { ignore_tags: %w[i18n] }.merge(options)
+      extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
+
+      extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
     end
 
     def options_for_html
@@ -34,7 +45,7 @@ module I18n::Tasks::Translators
     end
 
     def options_for_plain
-      { preserve_formatting: true }
+      { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
     end
 
     # @param [String] value
@@ -60,22 +71,34 @@ module I18n::Tasks::Translators
 
     private
 
-    # Convert 'es-ES' to 'ES'
-    def to_deepl_compatible_locale(locale)
+    # Convert 'es-ES' to 'ES', en-us to EN
+    def to_deepl_source_locale(locale)
       locale.to_s.split('-', 2).first.upcase
     end
 
+    # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
+    def to_deepl_target_locale(locale)
+      loc, sub = locale.to_s.split('-')
+      if SPECIFIC_TARGETS.include?(loc)
+        # Must see how the deepl api evolves, so this could be an error in the future
+        warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
+        locale.to_s.upcase
+      else
+        loc.upcase
+      end
+    end
+
     def configure_api_key!
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]
       version = @i18n_tasks.translation_config[:deepl_version]
       fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.no_api_key') if api_key.blank?
 
-      DeepL.configure { |config|
+      DeepL.configure do |config|
         config.auth_key = api_key
         config.host = host unless host.blank?
         config.version = version unless version.blank?
-      }
+      end
     end
   end
 end

data/lib/i18n/tasks/translators/google_translator.rb

@@ -55,7 +55,7 @@ module I18n::Tasks::Translators
         key = @i18n_tasks.translation_config[:google_translate_api_key]
         # fallback with deprecation warning
         if @i18n_tasks.translation_config[:api_key]
-          @i18n_tasks.warn_deprecated(
+          warn_deprecated(
             'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
           )
           key ||= translation_config[:api_key]