i18n-tasks 1.0.12 → 1.0.14

data/lib/i18n/tasks/references.rb

@@ -90,9 +90,9 @@ module I18n::Tasks
           on_leaves_merge: lambda do |node, other|
             if node.value != other.value
               log_warn(
-                'Conflicting references: '\
-                "#{node.full_key(root: false)} ⮕ #{node.value} in #{node.data[:locale]},"\
-                " but ⮕ #{other.value} in #{other.data[:locale]}"
+                'Conflicting references: ' \
+                "#{node.full_key(root: false)} ⮕ #{node.value} in #{node.data[:locale]}, " \
+                "but ⮕ #{other.value} in #{other.data[:locale]}"
               )
             end
           end

data/lib/i18n/tasks/reports/base.rb

@@ -36,7 +36,7 @@ module I18n::Tasks::Reports
 
     def used_title(keys_nodes, filter)
       used_n = keys_nodes.map { |_k, node| node.data[:occurrences].size }.reduce(:+).to_i
-      "#{keys_nodes.size} key#{'s' if keys_nodes.size != 1}#{" matching '#{filter}'" if filter}"\
+      "#{keys_nodes.size} key#{'s' if keys_nodes.size != 1}#{" matching '#{filter}'" if filter}" \
         "#{" (#{used_n} usage#{'s' if used_n != 1})" if used_n.positive?}"
     end
 

data/lib/i18n/tasks/reports/terminal.rb

@@ -93,6 +93,12 @@ module I18n
           end
         end
 
+        def cp_results(results)
+          results.each do |(from, to)|
+            print_info "#{Rainbow(from).cyan} #{Rainbow('+').yellow.bright} #{Rainbow(to).green}"
+          end
+        end
+
         def check_normalized_results(non_normalized)
           if non_normalized.empty?
             print_success 'All data is normalized'
@@ -112,7 +118,7 @@ module I18n
           when :missing_plural
             leaf[:data][:missing_keys].join(', ')
           else
-            "#{Rainbow(leaf[:data][:missing_diff_locale]).cyan} "\
+            "#{Rainbow(leaf[:data][:missing_diff_locale]).cyan} " \
             "#{format_value(leaf[:value].is_a?(String) ? leaf[:value].strip : leaf[:value])}"
           end
         end

data/lib/i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/scanners/ast_matchers/base_matcher'
+require 'i18n/tasks/scanners/results/occurrence'
+
+module I18n::Tasks::Scanners::AstMatchers
+  class DefaultI18nSubjectMatcher < BaseMatcher
+    def convert_to_key_occurrences(send_node, method_name, location: send_node.loc)
+      children = Array(send_node&.children)
+      return unless children[1] == :default_i18n_subject
+
+      key = @scanner.absolute_key(
+        '.subject',
+        location.expression.source_buffer.name,
+        calling_method: method_name
+      )
+      [
+        key,
+        I18n::Tasks::Scanners::Results::Occurrence.from_range(
+          raw_key: key,
+          range: location.expression
+        )
+      ]
+    end
+  end
+end

data/lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb

@@ -79,10 +79,10 @@ module I18n::Tasks::Scanners::AstMatchers
       end
       if default_arg_node = extract_hash_pair(node, 'default')
         default_arg = if default_arg_node.children[1]&.type == :hash
-          extract_hash(default_arg_node.children[1])
-        else
-          extract_string(default_arg_node.children[1])
-        end
+                        extract_hash(default_arg_node.children[1])
+                      else
+                        extract_string(default_arg_node.children[1])
+                      end
       end
 
       [key, default_arg]

data/lib/i18n/tasks/scanners/erb_ast_scanner.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 require 'i18n/tasks/scanners/ruby_ast_scanner'
-require 'i18n/tasks/scanners/erb_ast_processor'
-require 'better_html/errors'
-require 'better_html/parser'
+require 'i18n/tasks/scanners/local_ruby_parser'
 
 module I18n::Tasks::Scanners
   # Scan for I18n.translate calls in ERB-file better-html and ASTs
   class ErbAstScanner < RubyAstScanner
+    DEFAULT_REGEXP = /<%(={1,2}|-|\#|%)?(.*?)([-=])?%>/m.freeze
+
     def initialize(**args)
       super(**args)
-      @erb_ast_processor = ErbAstProcessor.new
+      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
     end
 
     private
@@ -20,29 +20,59 @@ module I18n::Tasks::Scanners
     # @param path Path to file to parse
     # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
     def path_to_ast_and_comments(path)
-      parser = BetterHtml::Parser.new(make_buffer(path))
-      ast = convert_better_html(parser.ast)
-      @erb_ast_processor.process_and_extract_comments(ast)
+      comments = []
+      buffer = make_buffer(path)
+
+      children = []
+      buffer
+        .source
+        .scan(DEFAULT_REGEXP) do |indicator, code, tailch, _rspace|
+          match = Regexp.last_match
+          character = indicator ? indicator[0] : nil
+
+          start = match.begin(0) + 2 + (character&.size || 0)
+          stop = match.end(0) - 2 - (tailch&.size || 0)
+
+          case character
+          when '=', nil, '-'
+            parsed, parsed_comments = handle_code(buffer, code, start, stop)
+            comments.concat(parsed_comments)
+            children << parsed unless parsed.nil?
+          when '#', '#-'
+            comments << handle_comment(buffer, start, stop)
+          end
+        end
+
+      [root_node(children, buffer), comments]
     end
 
-    # Convert BetterHtml nodes to Parser::AST::Node
-    #
-    # @param node BetterHtml::Parser::AST::Node
-    # @return Parser::AST::Node
-    def convert_better_html(node)
-      definition = Parser::Source::Map::Definition.new(
-        node.location.begin,
-        node.location.begin,
-        node.location.begin,
-        node.location.end
-      )
-      Parser::AST::Node.new(
-        node.type,
-        node.children.map { |child| child.is_a?(BetterHtml::AST::Node) ? convert_better_html(child) : child },
-        {
-          location: definition
-        }
-      )
+    def handle_code(buffer, code, start, stop)
+      range = ::Parser::Source::Range.new(buffer, start, stop)
+      location =
+        Parser::Source::Map::Definition.new(
+          range.begin,
+          range.begin,
+          range.begin,
+          range.end
+        )
+      @ruby_parser.parse(code, location: location)
+    end
+
+    def handle_comment(buffer, start, stop)
+      range = ::Parser::Source::Range.new(buffer, start, stop)
+      ::Parser::Source::Comment.new(range)
+    end
+
+    def root_node(children, buffer)
+      range = ::Parser::Source::Range.new(buffer, 0, buffer.source.size)
+      location =
+        Parser::Source::Map::Definition.new(
+          range.begin,
+          range.begin,
+          range.begin,
+          range.end
+        )
+      ::Parser::AST::Node.new(:erb, children, location: location)
     end
   end
 end

data/lib/i18n/tasks/scanners/pattern_scanner.rb

@@ -12,7 +12,7 @@ module I18n::Tasks::Scanners
     include OccurrenceFromPosition
     include RubyKeyLiterals
 
-    TRANSLATE_CALL_RE = /(?<=^|[^\w'\-.]|[^\w'\-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
+    TRANSLATE_CALL_RE = /(?<=^|[^\p{L}'\-.]|[^\p{L}'-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
     IGNORE_LINES = {
       'coffee' => /^\s*#(?!\si18n-tasks-use)/,
       'erb' => /^\s*<%\s*#(?!\si18n-tasks-use)/,

data/lib/i18n/tasks/scanners/relative_keys.rb

@@ -61,7 +61,7 @@ module I18n
             "#{file_key.sub(/_controller$/, '')}.#{calling_method}"
           else
             # Remove _ prefix from partials
-            file_key.gsub(/\._/, DOT)
+            file_key.gsub('._', DOT)
           end
         end
       end

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -3,7 +3,9 @@
 require 'i18n/tasks/scanners/file_scanner'
 require 'i18n/tasks/scanners/relative_keys'
 require 'i18n/tasks/scanners/ruby_ast_call_finder'
+require 'i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher'
 require 'i18n/tasks/scanners/ast_matchers/message_receivers_matcher'
+require 'i18n/tasks/scanners/ast_matchers/rails_model_matcher'
 require 'parser/current'
 
 module I18n::Tasks::Scanners
@@ -79,7 +81,7 @@ module I18n::Tasks::Scanners
         results = []
 
         # method_name is not available at this stage
-        calls.each do |send_node, _method_name|
+        calls.each do |(send_node, _method_name)|
           @matchers.each do |matcher|
             result = matcher.convert_to_key_occurrences(
               send_node,

data/lib/i18n/tasks/split_key.rb

@@ -5,6 +5,11 @@ module I18n
     module SplitKey
       module_function
 
+      PARENTHESIS_PAIRS = %w({} [] () <>).freeze
+      START_KEYS = PARENTHESIS_PAIRS.to_set { |pair| pair[0] }.freeze
+      END_KEYS = PARENTHESIS_PAIRS.to_h { |pair| [pair[0], pair[1]] }.freeze
+      private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS
+
       # split a key by dots (.)
       # dots inside braces or parenthesis are not split on
       #
@@ -12,61 +17,39 @@ module I18n
       # split_key 'a.#{b.c}' # => ['a', '#{b.c}']
       # split_key 'a.b.c', 2 # => ['a', 'b.c']
       def split_key(key, max = Float::INFINITY)
-        parts = []
-        pos   = 0
         return [key] if max == 1
 
-        key_parts(key) do |part|
-          parts << part
-          pos += part.length + 1
-          if parts.length + 1 >= max
-            parts << key[pos..] unless pos == key.length
-            break
+        parts = []
+        current_parenthesis_end_char = nil
+        part = ''
+        key.each_char.with_index do |char, index|
+          if current_parenthesis_end_char
+            part += char
+            current_parenthesis_end_char = nil if char == current_parenthesis_end_char
+          elsif START_KEYS.include?(char)
+            part += char
+            current_parenthesis_end_char = END_KEYS[char]
+          elsif char == '.'
+            parts << part
+            if parts.size + 1 == max
+              remaining = key[(index + 1)..]
+              parts << remaining unless remaining.empty?
+              return parts
+            end
+            part = ''
+          else
+            part += char
           end
         end
-        parts
-      end
-
-      def last_key_part(key)
-        last = nil
-        key_parts(key) { |part| last = part }
-        last
-      end
 
-      # yield each key part
-      # dots inside braces or parenthesis are not split on
-      def key_parts(key, &block)
-        return enum_for(:key_parts, key) unless block
+        return parts if part.empty?
 
-        nesting = PARENS
-        counts  = PARENS_ZEROS # dup'd later if key contains parenthesis
-        delim   = '.'
-        from    = to = 0
-        key.each_char do |char|
-          if char == delim && PARENS_ZEROS == counts
-            block.yield key[from...to]
-            from = to = (to + 1)
-          else
-            nest_i, nest_inc = nesting[char]
-            if nest_i
-              counts = counts.dup if counts.frozen?
-              counts[nest_i] += nest_inc
-            end
-            to += 1
-          end
-        end
-        block.yield(key[from...to]) if from < to && to <= key.length
-        true
+        current_parenthesis_end_char ? parts.concat(part.split('.')) : parts << part
       end
 
-      PARENS = %w({} [] ()).each_with_object({}) do |s, h|
-        i              = h.size / 2
-        h[s[0].freeze] = [i, 1].freeze
-        h[s[1].freeze] = [i, -1].freeze
-      end.freeze
-      PARENS_ZEROS = Array.new(PARENS.size, 0).freeze
-      private_constant :PARENS
-      private_constant :PARENS_ZEROS
+      def last_key_part(key)
+        split_key(key).last
+      end
     end
   end
 end

data/lib/i18n/tasks/translation.rb

@@ -2,13 +2,14 @@
 
 require 'i18n/tasks/translators/deepl_translator'
 require 'i18n/tasks/translators/google_translator'
+require 'i18n/tasks/translators/openai_translator'
 require 'i18n/tasks/translators/yandex_translator'
 
 module I18n::Tasks
   module Translation
     # @param [I18n::Tasks::Tree::Siblings] forest to translate to the locales of its root nodes
     # @param [String] from locale
-    # @param [:deepl, :google, :yandex] backend
+    # @param [:deepl, :openai, :google, :yandex] backend
     # @return [I18n::Tasks::Tree::Siblings] translated forest
     def translate_forest(forest, from:, backend: :google)
       case backend
@@ -16,6 +17,8 @@ module I18n::Tasks
         Translators::DeeplTranslator.new(self).translate_forest(forest, from)
       when :google
         Translators::GoogleTranslator.new(self).translate_forest(forest, from)
+      when :openai
+        Translators::OpenAiTranslator.new(self).translate_forest(forest, from)
       when :yandex
         Translators::YandexTranslator.new(self).translate_forest(forest, from)
       else

data/lib/i18n/tasks/translators/base_translator.rb

@@ -3,6 +3,7 @@
 module I18n::Tasks
   module Translators
     class BaseTranslator
+      include ::I18n::Tasks::Logging
       # @param [I18n::Tasks::BaseTask] i18n_tasks
       def initialize(i18n_tasks)
         @i18n_tasks = i18n_tasks
@@ -31,7 +32,7 @@ module I18n::Tasks
         reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
         list -= reference_key_vals
         result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
-          fetch_translations list_slice, opts.merge(is_html ? options_for_html : options_for_plain)
+          fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
         end.reduce(:+) || []
         result.concat(reference_key_vals)
         result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
@@ -41,34 +42,39 @@ module I18n::Tasks
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<[String, Object]>] translated list
       def fetch_translations(list, opts)
-        from_values(list, translate_values(to_values(list), **options_for_translate_values(**opts))).tap do |result|
+        options = options_for_translate_values(**opts)
+        from_values(list, translate_values(to_values(list, options), **options), options).tap do |result|
           fail CommandError, no_results_error_message if result.blank?
         end
       end
 
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<String>] values for translation extracted from list
-      def to_values(list)
-        list.map { |l| dump_value l[1] }.flatten.compact
+      def to_values(list, opts)
+        list.map { |l| dump_value(l[1], opts) }.flatten.compact
       end
 
       # @param [Array<[String, Object]>] list
       # @param [Array<String>] translated_values
       # @return [Array<[String, Object]>] translated key-value pairs
-      def from_values(list, translated_values)
+      def from_values(list, translated_values, opts)
         keys = list.map(&:first)
         untranslated_values = list.map(&:last)
-        keys.zip parse_value(untranslated_values, translated_values.to_enum)
+        keys.zip parse_value(untranslated_values, translated_values.to_enum, opts)
       end
 
       # Prepare value for translation.
       # @return [String, Array<String, nil>, nil] value for Google Translate or nil for non-string values
-      def dump_value(value)
+      def dump_value(value, opts)
         case value
         when Array
           # dump recursively
-          value.map { |v| dump_value v }
+          value.map { |v| dump_value(v, opts) }
+        when Hash
+          # dump recursively
+          value.values.map { |v| dump_value(v, opts) }
         when String
+          value = CGI.escapeHTML(value) if opts[:html_escape]
           replace_interpolations value unless value.empty?
         end
       end
@@ -77,16 +83,21 @@ module I18n::Tasks
       # @param [Object] untranslated
       # @param [Enumerator] each_translated
       # @return [Object] final translated value
-      def parse_value(untranslated, each_translated)
+      def parse_value(untranslated, each_translated, opts)
         case untranslated
         when Array
           # implode array
-          untranslated.map { |from| parse_value(from, each_translated) }
+          untranslated.map { |from| parse_value(from, each_translated, opts) }
+        when Hash
+          # implode hash
+          untranslated.transform_values { |value| parse_value(value, each_translated, opts) }
         when String
           if untranslated.empty?
             untranslated
           else
-            restore_interpolations untranslated, each_translated.next
+            value = each_translated.next
+            value = CGI.unescapeHTML(value) if opts[:html_escape]
+            restore_interpolations(untranslated, value)
           end
         else
           untranslated

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -4,6 +4,11 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class DeeplTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    # those languages must be specified with their sub-kind e.g en-us
+    SPECIFIC_TARGETS = %w[en pt].freeze
+
     def initialize(*)
       begin
         require 'deepl'
@@ -17,16 +22,27 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, from:, to:, **options)
-      result = DeepL.translate(list, to_deepl_compatible_locale(from), to_deepl_compatible_locale(to), options)
-      if result.is_a?(DeepL::Resources::Text)
-        [result.text]
-      else
-        result.map(&:text)
+      results = []
+      list.each_slice(BATCH_SIZE) do |parts|
+        res = DeepL.translate(
+          parts,
+          to_deepl_source_locale(from),
+          to_deepl_target_locale(to),
+          options_with_glossary(options, from, to)
+        )
+        if res.is_a?(DeepL::Resources::Text)
+          results << res.text
+        else
+          results += res.map(&:text)
+        end
       end
+      results
     end
 
     def options_for_translate_values(**options)
-      { ignore_tags: %w[i18n] }.merge(options)
+      extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
+
+      extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
     end
 
     def options_for_html
@@ -34,7 +50,7 @@ module I18n::Tasks::Translators
     end
 
     def options_for_plain
-      { preserve_formatting: true }
+      { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
     end
 
     # @param [String] value
@@ -60,11 +76,23 @@ module I18n::Tasks::Translators
 
     private
 
-    # Convert 'es-ES' to 'ES'
-    def to_deepl_compatible_locale(locale)
+    # Convert 'es-ES' to 'ES', en-us to EN
+    def to_deepl_source_locale(locale)
       locale.to_s.split('-', 2).first.upcase
     end
 
+    # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
+    def to_deepl_target_locale(locale)
+      loc, sub = locale.to_s.split('-')
+      if SPECIFIC_TARGETS.include?(loc)
+        # Must see how the deepl api evolves, so this could be an error in the future
+        warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
+        locale.to_s.upcase
+      else
+        loc.upcase
+      end
+    end
+
     def configure_api_key!
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]
@@ -77,5 +105,26 @@ module I18n::Tasks::Translators
         config.version = version unless version.blank?
       end
     end
+
+    def options_with_glossary(options, from, to)
+      glossary = find_glossary(from, to)
+      glossary ? { glossary_id: glossary.id }.merge(options) : options
+    end
+
+    def all_ready_glossaries
+      @all_ready_glossaries ||= DeepL.glossaries.list
+    end
+
+    def find_glossary(from, to)
+      config_glossary_ids = @i18n_tasks.translation_config[:deepl_glossary_ids]
+      return unless config_glossary_ids
+
+      all_ready_glossaries.find do |glossary|
+        glossary.ready \
+          && glossary.source_lang == from \
+          && glossary.target_lang == to \
+          && config_glossary_ids.include?(glossary.id)
+      end
+    end
   end
 end

data/lib/i18n/tasks/translators/google_translator.rb

@@ -4,6 +4,7 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class GoogleTranslator < BaseTranslator
+    NEWLINE_PLACEHOLDER = '<br id=i18n />'
     def initialize(*)
       begin
         require 'easy_translate'
@@ -16,14 +17,21 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      EasyTranslate.translate(list, options)
+      restore_newlines(
+        EasyTranslate.translate(
+          replace_newlines_with_placeholder(list, options[:html]),
+          options,
+          format: :text
+        ),
+        options[:html]
+      )
     end
 
     def options_for_translate_values(from:, to:, **options)
       options.merge(
         api_key: api_key,
-        from: to_google_translate_compatible_locale(from),
-        to: to_google_translate_compatible_locale(to)
+        from: from,
+        to: to
       )
     end
 
@@ -41,21 +49,12 @@ module I18n::Tasks::Translators
 
     private
 
-    SUPPORTED_LOCALES_WITH_REGION = %w[zh-CN zh-TW].freeze
-
-    # Convert 'es-ES' to 'es'
-    def to_google_translate_compatible_locale(locale)
-      return locale unless locale.include?('-') && !SUPPORTED_LOCALES_WITH_REGION.include?(locale)
-
-      locale.split('-', 2).first
-    end
-
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:google_translate_api_key]
         # fallback with deprecation warning
         if @i18n_tasks.translation_config[:api_key]
-          @i18n_tasks.warn_deprecated(
+          warn_deprecated(
             'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
           )
           key ||= translation_config[:api_key]
@@ -65,5 +64,21 @@ module I18n::Tasks::Translators
         key
       end
     end
+
+    def replace_newlines_with_placeholder(list, html)
+      return list unless html
+
+      list.map do |value|
+        value.gsub("\n", NEWLINE_PLACEHOLDER)
+      end
+    end
+
+    def restore_newlines(translations, html)
+      return translations unless html
+
+      translations.map do |translation|
+        translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n")
+      end
+    end
   end
 end