i18n-tasks 1.0.13 → 1.0.15

data/lib/i18n/tasks/split_key.rb

@@ -5,6 +5,11 @@ module I18n
     module SplitKey
       module_function
 
+      PARENTHESIS_PAIRS = %w({} [] () <>).freeze
+      START_KEYS = PARENTHESIS_PAIRS.to_set { |pair| pair[0] }.freeze
+      END_KEYS = PARENTHESIS_PAIRS.to_h { |pair| [pair[0], pair[1]] }.freeze
+      private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS
+
       # split a key by dots (.)
       # dots inside braces or parenthesis are not split on
       #
@@ -12,61 +17,39 @@ module I18n
       # split_key 'a.#{b.c}' # => ['a', '#{b.c}']
       # split_key 'a.b.c', 2 # => ['a', 'b.c']
       def split_key(key, max = Float::INFINITY)
-        parts = []
-        pos   = 0
         return [key] if max == 1
 
-        key_parts(key) do |part|
-          parts << part
-          pos += part.length + 1
-          if parts.length + 1 >= max
-            parts << key[pos..] unless pos == key.length
-            break
+        parts = []
+        current_parenthesis_end_char = nil
+        part = ''
+        key.each_char.with_index do |char, index|
+          if current_parenthesis_end_char
+            part += char
+            current_parenthesis_end_char = nil if char == current_parenthesis_end_char
+          elsif START_KEYS.include?(char)
+            part += char
+            current_parenthesis_end_char = END_KEYS[char]
+          elsif char == '.'
+            parts << part
+            if parts.size + 1 == max
+              remaining = key[(index + 1)..]
+              parts << remaining unless remaining.empty?
+              return parts
+            end
+            part = ''
+          else
+            part += char
           end
         end
-        parts
-      end
-
-      def last_key_part(key)
-        last = nil
-        key_parts(key) { |part| last = part }
-        last
-      end
 
-      # yield each key part
-      # dots inside braces or parenthesis are not split on
-      def key_parts(key, &block)
-        return enum_for(:key_parts, key) unless block
+        return parts if part.empty?
 
-        nesting = PARENS
-        counts  = PARENS_ZEROS # dup'd later if key contains parenthesis
-        delim   = '.'
-        from    = to = 0
-        key.each_char do |char|
-          if char == delim && PARENS_ZEROS == counts
-            block.yield key[from...to]
-            from = to = (to + 1)
-          else
-            nest_i, nest_inc = nesting[char]
-            if nest_i
-              counts = counts.dup if counts.frozen?
-              counts[nest_i] += nest_inc
-            end
-            to += 1
-          end
-        end
-        block.yield(key[from...to]) if from < to && to <= key.length
-        true
+        current_parenthesis_end_char ? parts.concat(part.split('.')) : parts << part
       end
 
-      PARENS = %w({} [] ()).each_with_object({}) do |s, h|
-        i              = h.size / 2
-        h[s[0].freeze] = [i, 1].freeze
-        h[s[1].freeze] = [i, -1].freeze
-      end.freeze
-      PARENS_ZEROS = Array.new(PARENS.size, 0).freeze
-      private_constant :PARENS
-      private_constant :PARENS_ZEROS
+      def last_key_part(key)
+        split_key(key).last
+      end
     end
   end
 end

data/lib/i18n/tasks/translation.rb

@@ -3,6 +3,7 @@
 require 'i18n/tasks/translators/deepl_translator'
 require 'i18n/tasks/translators/google_translator'
 require 'i18n/tasks/translators/openai_translator'
+require 'i18n/tasks/translators/watsonx_translator'
 require 'i18n/tasks/translators/yandex_translator'
 
 module I18n::Tasks
@@ -11,7 +12,7 @@ module I18n::Tasks
     # @param [String] from locale
     # @param [:deepl, :openai, :google, :yandex] backend
     # @return [I18n::Tasks::Tree::Siblings] translated forest
-    def translate_forest(forest, from:, backend: :google)
+    def translate_forest(forest, from:, backend:)
       case backend
       when :deepl
         Translators::DeeplTranslator.new(self).translate_forest(forest, from)
@@ -19,6 +20,8 @@ module I18n::Tasks
         Translators::GoogleTranslator.new(self).translate_forest(forest, from)
       when :openai
         Translators::OpenAiTranslator.new(self).translate_forest(forest, from)
+      when :watsonx
+        Translators::WatsonxTranslator.new(self).translate_forest(forest, from)
       when :yandex
         Translators::YandexTranslator.new(self).translate_forest(forest, from)
       else

data/lib/i18n/tasks/translators/base_translator.rb

@@ -14,7 +14,11 @@ module I18n::Tasks
       # @return [I18n::Tasks::Tree::Siblings] translated forest
       def translate_forest(forest, from)
         forest.inject @i18n_tasks.empty_forest do |result, root|
-          translated = translate_pairs(root.key_values(root: true), to: root.key, from: from)
+          pairs = root.key_values(root: true)
+
+          @progress_bar = ProgressBar.create(total: pairs.flatten.size, format: '%a <%B> %e %c/%C (%p%%)')
+
+          translated = translate_pairs(pairs, to: root.key, from: from)
           result.merge! Data::Tree::Siblings.from_flat_pairs(translated)
         end
       end
@@ -70,6 +74,9 @@ module I18n::Tasks
         when Array
           # dump recursively
           value.map { |v| dump_value(v, opts) }
+        when Hash
+          # dump recursively
+          value.values.map { |v| dump_value(v, opts) }
         when String
           value = CGI.escapeHTML(value) if opts[:html_escape]
           replace_interpolations value unless value.empty?
@@ -85,6 +92,9 @@ module I18n::Tasks
         when Array
           # implode array
           untranslated.map { |from| parse_value(from, each_translated, opts) }
+        when Hash
+          # implode hash
+          untranslated.transform_values { |value| parse_value(value, each_translated, opts) }
         when String
           if untranslated.empty?
             untranslated

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -24,12 +24,19 @@ module I18n::Tasks::Translators
     def translate_values(list, from:, to:, **options)
       results = []
       list.each_slice(BATCH_SIZE) do |parts|
-        res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options)
+        res = DeepL.translate(
+          parts,
+          to_deepl_source_locale(from),
+          to_deepl_target_locale(to),
+          options_with_glossary(options, from, to)
+        )
         if res.is_a?(DeepL::Resources::Text)
           results << res.text
         else
           results += res.map(&:text)
         end
+
+        @progress_bar.progress += parts.size
       end
       results
     end
@@ -78,6 +85,9 @@ module I18n::Tasks::Translators
 
     # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
     def to_deepl_target_locale(locale)
+      locale_aliases = @i18n_tasks.translation_config[:deepl_locale_aliases]
+      locale = locale_aliases[locale.to_s.downcase] || locale if locale_aliases.is_a?(Hash)
+
       loc, sub = locale.to_s.split('-')
       if SPECIFIC_TARGETS.include?(loc)
         # Must see how the deepl api evolves, so this could be an error in the future
@@ -100,5 +110,26 @@ module I18n::Tasks::Translators
         config.version = version unless version.blank?
       end
     end
+
+    def options_with_glossary(options, from, to)
+      glossary = find_glossary(from, to)
+      glossary ? { glossary_id: glossary.id }.merge(options) : options
+    end
+
+    def all_ready_glossaries
+      @all_ready_glossaries ||= DeepL.glossaries.list
+    end
+
+    def find_glossary(from, to)
+      config_glossary_ids = @i18n_tasks.translation_config[:deepl_glossary_ids]
+      return unless config_glossary_ids
+
+      all_ready_glossaries.find do |glossary|
+        glossary.ready \
+          && glossary.source_lang == from \
+          && glossary.target_lang == to \
+          && config_glossary_ids.include?(glossary.id)
+      end
+    end
   end
 end

data/lib/i18n/tasks/translators/google_translator.rb

@@ -4,6 +4,7 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class GoogleTranslator < BaseTranslator
+    NEWLINE_PLACEHOLDER = '<br id=i18n />'
     def initialize(*)
       begin
         require 'easy_translate'
@@ -16,14 +17,25 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      EasyTranslate.translate(list, options)
+      result = restore_newlines(
+        EasyTranslate.translate(
+          replace_newlines_with_placeholder(list, options[:html]),
+          options,
+          format: options[:html] ? :html : :text
+        ),
+        options[:html]
+      )
+
+      @progress_bar.progress += result.size
+
+      result
     end
 
     def options_for_translate_values(from:, to:, **options)
       options.merge(
         api_key: api_key,
-        from: to_google_translate_compatible_locale(from),
-        to: to_google_translate_compatible_locale(to)
+        from: from,
+        to: to
       )
     end
 
@@ -41,15 +53,6 @@ module I18n::Tasks::Translators
 
     private
 
-    SUPPORTED_LOCALES_WITH_REGION = %w[zh-CN zh-TW].freeze
-
-    # Convert 'es-ES' to 'es'
-    def to_google_translate_compatible_locale(locale)
-      return locale unless locale.include?('-') && !SUPPORTED_LOCALES_WITH_REGION.include?(locale)
-
-      locale.split('-', 2).first
-    end
-
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:google_translate_api_key]
@@ -65,5 +68,25 @@ module I18n::Tasks::Translators
         key
       end
     end
+
+    def replace_newlines_with_placeholder(list, html)
+      return list unless html
+
+      list.map do |value|
+        value.gsub(/\n(\s*)/) do
+          "<Z__#{::Regexp.last_match(1)&.length || 0}>"
+        end
+      end
+    end
+
+    def restore_newlines(translations, html)
+      return translations unless html
+
+      translations.map do |translation|
+        translation.gsub(/<Z__(\d+)>/) do
+          "\n#{' ' * ::Regexp.last_match(1).to_i}"
+        end
+      end
+    end
   end
 end

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -1,11 +1,28 @@
 # frozen_string_literal: true
 
 require 'i18n/tasks/translators/base_translator'
+require 'active_support/core_ext/string/filters'
 
 module I18n::Tasks::Translators
   class OpenAiTranslator < BaseTranslator
     # max allowed texts per request
     BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a professional translator that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+
+      Keep in mind the context of all the strings for a more accurate translation.
+      It is CRITICAL you output only the result, without any additional information, code block syntax or comments.
+    PROMPT
+    JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT = <<~PROMPT.squish
+      Return the translations as a JSON object with a 'translations' array containing the translated strings.
+    PROMPT
 
     def initialize(*)
       begin
@@ -38,7 +55,7 @@ module I18n::Tasks::Translators
     private
 
     def translator
-      @translator ||= OpenAI::Client.new(access_token: api_key)
+      @translator ||= OpenAI::Client.new(access_token: api_key, log_errors: true)
     end
 
     def api_key
@@ -50,26 +67,56 @@ module I18n::Tasks::Translators
       end
     end
 
+    def model
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-4o-mini'
+    end
+
+    def system_prompt
+      @system_prompt ||=
+        (@i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT)
+        .concat("\n#{JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT}")
+      @system_prompt
+    end
+
     def translate_values(list, from:, to:)
       results = []
 
       list.each_slice(BATCH_SIZE) do |batch|
         translations = translate(batch, from, to)
+        result = JSON.parse(translations)
+        results << result
 
-        results << JSON.parse(translations)
+        @progress_bar.progress += result.size
       end
 
       results.flatten
     end
 
-    def translate(values, from, to) # rubocop:disable Metrics/MethodLength
-      messages = [
+    def translate(values, from, to)
+      response = translator.chat(
+        parameters: {
+          model: model,
+          messages: build_messages(values, from, to),
+          temperature: 0.0,
+          response_format: { type: 'json_object' }
+        }
+      )
+
+      translations = response.dig('choices', 0, 'message', 'content')
+      error = response['error']
+
+      fail "AI error: #{error}" if error.present?
+
+      # Extract the array from the JSON object response
+      result = JSON.parse(translations)
+      result['translations'].to_json
+    end
+
+    def build_messages(values, from, to)
+      [
         {
           role: 'system',
-          content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n
-          locale array. The array has a structured format and contains multiple strings. Your task is to translate
-          each of these strings and create a new array with the translated strings. Keep in mind the context of all
-          the strings for a more accurate translation.\n"
+          content: format(system_prompt, from: from, to: to)
         },
         {
           role: 'user',
@@ -80,21 +127,6 @@ module I18n::Tasks::Translators
           content: values.to_json
         }
       ]
-
-      response = translator.chat(
-        parameters: {
-          model: 'gpt-3.5-turbo',
-          messages: messages,
-          temperature: 0.7
-        }
-      )
-
-      translations = response.dig('choices', 0, 'message', 'content')
-      error = response['error']
-
-      fail "AI error: #{error}" if error.present?
-
-      translations
     end
   end
 end

data/lib/i18n/tasks/translators/watsonx_translator.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/translators/base_translator'
+require 'active_support/core_ext/string/filters'
+
+module I18n::Tasks::Translators
+  class WatsonxTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a helpful assistant that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+      You always preserve the structure and formatting exactly as it is.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      Reminder:
+      - Translate only the text, preserving the structure and formatting.
+      - Do not translate any URLs.
+      - Do not translate HTML tags like `<details>` and `<summary>`.
+      - HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      - Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+      - Output only the result, without any additional information or comments.
+    PROMPT
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t('i18n_tasks.watsonx_translate.errors.no_results')
+    end
+
+    private
+
+    def translator
+      @translator ||= WatsonxClient.new(key: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:watsonx_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.watsonx_translate.errors.no_api_key') if key.blank?
+
+        key
+      end
+    end
+
+    def project_id
+      @project_id ||= begin
+        project_id = @i18n_tasks.translation_config[:watsonx_project_id]
+        if project_id.blank?
+          fail ::I18n::Tasks::CommandError,
+               I18n.t('i18n_tasks.watsonx_translate.errors.no_project_id')
+        end
+
+        project_id
+      end
+    end
+
+    def model
+      @model ||= @i18n_tasks.translation_config[:watsonx_model].presence || 'meta-llama/llama-3-2-90b-vision-instruct'
+    end
+
+    def system_prompt
+      @system_prompt ||= @i18n_tasks.translation_config[:watsonx_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+        result = JSON.parse(translations)
+        results << result
+
+        @progress_bar.progress += results.size
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to)
+      prompt = [
+        '<|eot_id|><|start_header_id|>system<|end_header_id|>',
+        format(system_prompt, from: from, to: to),
+        '<|eot_id|><|start_header_id|>user<|end_header_id|>Translate this array:',
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>#{values.to_json}",
+        '<|eot_id|><|start_header_id|>assistant<|end_header_id|>'
+      ].join
+
+      response = translator.generate_text(
+        model_id: model,
+        project_id: project_id,
+        input: prompt,
+        parameters: {
+          decoding_method: :greedy,
+          max_new_tokens: 2048,
+          repetition_penalty: 1
+        }
+      )
+      response.dig('results', 0, 'generated_text')
+    end
+  end
+end
+
+class WatsonxClient
+  WATSONX_BASE_URL = 'https://us-south.ml.cloud.ibm.com/ml/'
+  IBM_CLOUD_IAM_URL = 'https://iam.cloud.ibm.com/identity/token'
+
+  def initialize(key:)
+    begin
+      require 'faraday'
+    rescue LoadError
+      raise ::I18n::Tasks::CommandError, "Add gem 'faraday' to your Gemfile to use this command"
+    end
+
+    @http = Faraday.new(url: WATSONX_BASE_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.request :json
+      conn.response :json
+      conn.options.timeout = 600
+      conn.request :authorization, :Bearer, token(key)
+    end
+  end
+
+  def generate_text(**opts)
+    @http.post('v1/text/generation?version=2024-05-20', **opts).body
+  end
+
+  private
+
+  def token(key)
+    Faraday.new(url: IBM_CLOUD_IAM_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.response :json
+      conn.params = {
+        grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
+        apikey: key
+      }
+    end.post.body['access_token']
+  end
+end

data/lib/i18n/tasks/translators/yandex_translator.rb

@@ -16,7 +16,11 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      list.map { |item| translator.translate(item, options) }
+      result = list.map { |item| translator.translate(item, options) }
+
+      @progress_bar.progress += result.size
+
+      result
     end
 
     def options_for_translate_values(from:, to:, **options)

data/lib/i18n/tasks/used_keys.rb

@@ -4,6 +4,7 @@ require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
 require 'i18n/tasks/scanners/erb_ast_scanner'
+require 'i18n/tasks/scanners/prism_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '1.0.13'
+    VERSION = '1.0.15'
   end
 end

data/lib/i18n/tasks.rb

@@ -66,6 +66,7 @@ require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/try'
 
+require 'ruby-progressbar'
 require 'rainbow'
 require 'erubi'
 

data/templates/config/i18n-tasks.yml

@@ -13,7 +13,7 @@ data:
   ## Provide a custom adapter:
   # adapter: I18n::Tasks::Data::FileSystem
 
-  # Locale files or `Find.find` patterns where translations are read from:
+  # Locale files or `Dir.glob` patterns where translations are read from:
   read:
     ## Default:
     # - config/locales/%{locale}.yml
@@ -34,7 +34,7 @@ data:
     ## Example (replace %#= with %=):
     # - "<%#= %x[bundle info vagrant --path].chomp %>/templates/locales/%{locale}.yml"
 
-  ## Specify the router (see Readme for details). Valid values: conservative_router, pattern_router, or a custom class.
+  ## Specify the router (see Readme for details). Valid values: conservative_router, pattern_router, isolating_router, or a custom class.
   # router: conservative_router
 
   yaml:
@@ -92,9 +92,13 @@ search:
   ## - RailsModelMatcher
   ##     Matches ActiveRecord translations like
   ##     User.human_attribute_name(:email) and User.model_name.human
+  ## - DefaultI18nSubjectMatcher
+  ##     Matches ActionMailer's default_i18n_subject method
   ##
   ## To implement your own, please see `I18n::Tasks::Scanners::AstMatchers::BaseMatcher`.
-  # <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
+  # ast_matchers:
+  #   - 'I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher'
+  #   - 'I18n::Tasks::Scanners::AstMatchers::DefaultI18nSubjectMatcher'
 
   ## Multiple scanners can be used. Their results are merged.
   ## The options specified above are passed down to each scanner. Per-scanner options can be specified as well.
@@ -110,9 +114,28 @@ search:
 #   deepl_api_key: "48E92789-57A3-466A-9959-1A1A1A1A1A1A"
 #   # deepl_host: "https://api.deepl.com"
 #   # deepl_version: "v2"
+#   # deepl_glossary_ids:
+#   #   - f28106eb-0e06-489e-82c6-8215d6f95089
+#   #   - 2c6415be-1852-4f54-9e1b-d800463496b4
 #   # add additional options to the DeepL.translate call: https://www.deepl.com/docs-api/translate-text/translate-text/
 #   deepl_options:
 #     formality: prefer_less
+#   # OpenAI
+#   openai_api_key: "sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+#   # openai_model: "gpt-4o-mini" # see https://platform.openai.com/docs/models
+#   # may contain `%{from}` and `%{to}`, which will be replaced by source and target locale codes, respectively (using `Kernel.format`)
+#   # openai_system_prompt: >-
+#   #   You are a professional translator that translates content from the %{from} locale
+#   #   to the %{to} locale in an i18n locale array.
+#   #
+#   #   The array has a structured format and contains multiple strings. Your task is to translate
+#   #   each of these strings and create a new array with the translated strings.
+#   #
+#   #   HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+#   #   Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+#   #
+#   #   Keep in mind the context of all the strings for a more accurate translation.
+
 ## Do not consider these keys missing:
 # ignore_missing:
 # - 'errors.messages.{accepted,blank,invalid,too_short,too_long}'