i18n-tasks 1.0.11 → 1.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76a7092606f1ea327f57a9a2bc0489c07d5349f61b6b742415d0dd367b243237
-  data.tar.gz: dd87c3098d774df0fe685829cefe3716e9cd42156cb483762a491174d0eaf733
+  metadata.gz: 12e19e4d7fe61bd20e000b0b6e4db5bdf233443e8e8cf965b62e634bdfa3e5a2
+  data.tar.gz: 3dab05338d079c4defad590466ab2985faf43245d616f86af4c212003831a6e4
 SHA512:
-  metadata.gz: 7341b68328712adcbbb7b28b9c83695a56fcaac29eb9d7303c8866cf0f6004bd7e85065a1684a3acb1597786c1993b876e8eab04f5db555d95563f43976e7934
-  data.tar.gz: 8b8f93c545f0cc80b3ee36610751db469c1cf6c47df4ba9376189c926480dc0a96d678545c90163f2f24f1826beeaaff1e24ec9bc273a24541e715b0701ceb61
+  metadata.gz: 2cf99aec1c6d5738c4a71964a7ce67e70c8ade3150042c4e381fef19b9eaa82cf6638979bf97c6e9941f119304973f9e6f3fd31b3bab0fd635520344448b5587
+  data.tar.gz: 2de5ab47951585b298e43bece202f3ad885ae8eb03edb2b6e860b30e3172ea213947509b9543bbf31df06ef03079ff1e7cfd1b7a32cd964428d9f9612fd7e171

data/README.md

@@ -24,7 +24,7 @@ i18n-tasks can be used with any project using the ruby [i18n gem][i18n-gem] (def
 Add i18n-tasks to the Gemfile:
 
 ```ruby
-gem 'i18n-tasks', '~> 1.0.11'
+gem 'i18n-tasks', '~> 1.0.13'
 ```
 
 Copy the default [configuration file](#configuration):
@@ -117,6 +117,17 @@ $ i18n-tasks translate-missing --backend=yandex
 $ i18n-tasks translate-missing --from=en es fr
 ```
 
+### OpenAI Translate missing keys
+
+Translate missing values with OpenAI ([more below on the API key](#openai-translation-config)).
+
+```console
+$ i18n-tasks translate-missing --backend=openai
+
+# accepts from and locales options:
+$ i18n-tasks translate-missing --from=en es fr
+```
+
 ### Find usages
 
 See where the keys are used with `i18n-tasks find`:
@@ -335,6 +346,7 @@ A special syntax similar to file glob patterns is used throughout i18n-tasks to
 |:------------:|:----------------------------------------------------------|
 |      `*`     | matches everything                                        |
 |      `:`     | matches a single key                                      |
+|      `*:`    | matches part of a single key                              |
 |   `{a, b.c}` | match any in set, can use `:` and `*`, match is captured  |
 
 Example of usage:
@@ -423,6 +435,17 @@ translation:
   yandex_api_key: <Yandex API key>
 ```
 
+<a name="openai-translation-config"></a>
+### OpenAI Translate
+
+`i18n-tasks translate-missing` requires a OpenAI API key, get it at [OpenAI](https://openai.com/).
+
+```yaml
+# config/i18n-tasks.yml
+translation:
+  openai_api_key: <OpenAI API key>
+```
+
 ## Interactive console
 
 `i18n-tasks irb` starts an IRB session in i18n-tasks context. Type `guide` for more information.
@@ -440,7 +463,7 @@ Custom tasks can be added easily, see the examples [on the wiki](https://github.
 
 - Install dependencies using `bundle install`
 - Run tests using `bundle exec rspec`
-- Install [Overcommit](overcommit) by running `overcommit --install`
+- Install [Overcommit](https://github.com/sds/overcommit) by running `overcommit --install`
 
 ## Skip Overcommit-hooks
 

data/Rakefile

@@ -9,5 +9,5 @@ task :irb do
   # $: << File.expand_path('lib', __FILE__)
   require 'i18n/tasks'
   require 'i18n/tasks/commands'
-  ::I18n::Tasks::Commands.new(::I18n::Tasks::BaseTask.new).irb
+  I18n::Tasks::Commands.new(I18n::Tasks::BaseTask.new).irb
 end

data/config/locales/en.yml

@@ -33,7 +33,7 @@ en:
             Value. Interpolates: %{value}, %{human_key}, %{key}, %{default}, %{value_or_human_key},
             %{value_or_default_or_human_key}
       desc:
-        add_missing: add missing keys to locale data
+        add_missing: add missing keys to locale data, optionally match a pattern
         check_consistent_interpolations: verify that all translations use correct interpolation variables
         check_normalized: verify that all translation data is normalized
         config: display i18n-tasks configuration
@@ -46,12 +46,13 @@ en:
         gem_path: show path to the gem
         health: is everything OK?
         irb: start REPL session within i18n-tasks context
-        missing: show missing translations
+        missing: show missing translations, optionally match a pattern
         mv: rename/merge the keys in locale data that match the given pattern
         normalize: 'normalize translation data: sort and move to the right files'
         remove_unused: remove unused keys
         rm: remove the keys in locale data that match the given pattern
-        translate_missing: translate missing keys with Google Translate or DeepL Pro
+        translate_missing: translate missing keys with Google Translate or DeepL Pro, optionally match
+          a pattern
         tree_convert: convert tree between formats
         tree_filter: filter tree by key pattern
         tree_merge: merge trees
@@ -95,6 +96,8 @@ en:
           Setup DeepL Pro API key via DEEPL_AUTH_KEY environment variable or translation.deepl_api_key
           in config/i18n-tasks.yml. Get the key at https://www.deepl.com/pro.
         no_results: DeepL returned no results.
+        specific_target_missing: You must supply a specific variant for the given target language
+          e.g. en-us instead of en.
     google_translate:
       errors:
         no_api_key: >-
@@ -109,6 +112,12 @@ en:
     missing:
       details_title: Value in other locales or source
       none: No translations are missing.
+    openai_translate:
+      errors:
+        no_api_key: >-
+          Set OpenAI API key via OPENAI_API_KEY environment variable or translation.openai_api_key
+          in config/i18n-tasks.yml. Get the key at https://openai.com/.
+        no_results: OpenAI returned no results.
     remove_unused:
       confirm:
         one: "%{count} translation will be removed from %{locales}."

data/config/locales/ru.yml

@@ -94,6 +94,8 @@ ru:
           Задайте ключ API DeepL через переменную окружения DEEPL_AUTH_KEY или translation.deepl_api_key
           Получите ключ через https://www.deepl.com/pro.
         no_results: DeepL не дал результатов.
+        specific_target_missing: You must supply a specific variant for the given target language
+          e.g. en-us instead of en.
     google_translate:
       errors:
         no_api_key: >-
@@ -109,6 +111,12 @@ ru:
     missing:
       details_title: На других языках или в коде
       none: Всё переведено.
+    openai_translate:
+      errors:
+        no_api_key: |-
+          Установить ключ API Яндекса с помощью переменной среды OPENAI_API_KEY или translation.openai_api_key
+           в config / i18n-tasks.yml. Получите ключ по адресу https://openai.com/.
+        no_results: Яндекс не дал результатов.
     remove_unused:
       confirm:
         few: Переводы (%{count}) будут удалены из %{locales}.

data/i18n-tasks.gemspec

@@ -4,7 +4,7 @@ lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'i18n/tasks/version'
 
-Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |s|
   s.name          = 'i18n-tasks'
   s.version       = I18n::Tasks::VERSION
   s.authors       = ['glebm']
@@ -35,16 +35,15 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
   s.files = `git ls-files`.split($/)
   s.files -= s.files.grep(%r{^(doc/|\.|spec/)}) + %w[CHANGES.md config/i18n-tasks.yml Gemfile]
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) } - %w[i18n-tasks.cmd]
-  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ['lib']
 
   s.add_dependency 'activesupport', '>= 4.0.2'
   s.add_dependency 'ast', '>= 2.1.0'
-  s.add_dependency 'better_html', '~> 1.0'
+  s.add_dependency 'better_html', '>= 1.0', '< 3.0'
   s.add_dependency 'erubi'
   s.add_dependency 'highline', '>= 2.0.0'
   s.add_dependency 'i18n'
-  s.add_dependency 'parser', '>= 2.2.3.0'
+  s.add_dependency 'parser', '>= 3.2.2.1'
   s.add_dependency 'rails-i18n'
   s.add_dependency 'rainbow', '>= 2.2.2', '< 4.0'
   s.add_dependency 'terminal-table', '>= 1.5.1'
@@ -52,7 +51,9 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
   s.add_development_dependency 'overcommit', '~> 0.58.0'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rspec', '~> 3.3'
-  s.add_development_dependency 'rubocop', '~> 1.27.0'
+  s.add_development_dependency 'rubocop', '~> 1.50.1'
+  s.add_development_dependency 'rubocop-rake', '~> 0.6.0'
+  s.add_development_dependency 'rubocop-rspec', '~> 2.19.0'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'yard'
 

data/lib/i18n/tasks/cli.rb

@@ -35,14 +35,14 @@ class I18n::Tasks::CLI
 
   def run(argv)
     argv.each_with_index do |arg, i|
-      if ['--config', '-c'].include?(arg)
-        _, config_file = argv.slice!(i, 2)
-        if File.exist?(config_file)
-          @config_file = config_file
-          break
-        else
-          error "Config file doesn't exist: #{config_file}", 128
-        end
+      next unless ['--config', '-c'].include?(arg)
+
+      _, config_file = argv.slice!(i, 2)
+      if File.exist?(config_file)
+        @config_file = config_file
+        break
+      else
+        error "Config file doesn't exist: #{config_file}", 128
       end
     end
 

data/lib/i18n/tasks/command/commands/missing.rb

@@ -25,10 +25,14 @@ module I18n::Tasks
         cmd :missing,
             pos: '[locale ...]',
             desc: t('i18n_tasks.cmd.desc.missing'),
-            args: %i[locales out_format missing_types]
+            args: %i[locales out_format missing_types pattern]
 
         def missing(opt = {})
           forest = i18n.missing_keys(**opt.slice(:locales, :base_locale, :types))
+          if opt[:pattern]
+            pattern_re = i18n.compile_key_pattern(opt[:pattern])
+            forest.select_keys! { |full_key, _node| full_key =~ pattern_re }
+          end
           print_forest forest, opt, :missing_keys
           :exit1 unless forest.empty?
         end
@@ -36,10 +40,14 @@ module I18n::Tasks
         cmd :translate_missing,
             pos: '[locale ...]',
             desc: t('i18n_tasks.cmd.desc.translate_missing'),
-            args: [:locales, :locale_to_translate_from, arg(:out_format).from(1), :translation_backend]
+            args: [:locales, :locale_to_translate_from, arg(:out_format).from(1), :translation_backend, :pattern]
 
         def translate_missing(opt = {})
-          missing    = i18n.missing_diff_forest opt[:locales], opt[:from]
+          missing = i18n.missing_diff_forest opt[:locales], opt[:from]
+          if opt[:pattern]
+            pattern_re = i18n.compile_key_pattern(opt[:pattern])
+            missing.select_keys! { |full_key, _node| full_key =~ pattern_re }
+          end
           translated = i18n.translate_forest missing, from: opt[:from], backend: opt[:backend].to_sym
           i18n.data.merge! translated
           log_stderr t('i18n_tasks.translate_missing.translated', count: translated.leaves.count)
@@ -49,17 +57,21 @@ module I18n::Tasks
         cmd :add_missing,
             pos: '[locale ...]',
             desc: t('i18n_tasks.cmd.desc.add_missing'),
-            args: [:locales, :out_format, arg(:value) + [{ default: '%{value_or_default_or_human_key}' }],
+            args: [:locales, :out_format, :pattern, arg(:value) + [{ default: '%{value_or_default_or_human_key}' }],
                    ['--nil-value', 'Set value to nil. Takes precedence over the value argument.']]
 
         # Merge base locale first, as this may affect the value for the other locales
-        def add_missing(opt = {})
+        def add_missing(opt = {}) # rubocop:disable Metrics/AbcSize
           [
             [i18n.base_locale] & opt[:locales],
             opt[:locales] - [i18n.base_locale]
           ].reject(&:empty?).each_with_object(i18n.empty_forest) do |locales, added|
             forest = i18n.missing_keys(locales: locales, **opt.slice(:types, :base_locale))
                          .set_each_value!(opt[:'nil-value'] ? nil : opt[:value])
+            if opt[:pattern]
+              pattern_re = i18n.compile_key_pattern(opt[:pattern])
+              forest.select_keys! { |full_key, _node| full_key =~ pattern_re }
+            end
             i18n.data.merge! forest
             added.merge! forest
           end.tap do |added|

data/lib/i18n/tasks/configuration.rb

@@ -66,6 +66,7 @@ module I18n::Tasks::Configuration # rubocop:disable Metrics/ModuleLength
       conf[:deepl_api_key] = ENV['DEEPL_AUTH_KEY'] if ENV.key?('DEEPL_AUTH_KEY')
       conf[:deepl_host] = ENV['DEEPL_HOST'] if ENV.key?('DEEPL_HOST')
       conf[:deepl_version] = ENV['DEEPL_VERSION'] if ENV.key?('DEEPL_VERSION')
+      conf[:openai_api_key] = ENV['OPENAI_API_KEY'] if ENV.key?('OPENAI_API_KEY')
       conf[:yandex_api_key] = ENV['YANDEX_API_KEY'] if ENV.key?('YANDEX_API_KEY')
       conf
     end
@@ -87,7 +88,7 @@ module I18n::Tasks::Configuration # rubocop:disable Metrics/ModuleLength
       valid_locales = Dir[File.join(I18n::Tasks.gem_path, 'config', 'locales', '*.yml')]
                       .map { |f| File.basename(f, '.yml') }
       unless valid_locales.include?(internal_locale)
-        log_warn "invalid internal_locale #{internal_locale.inspect}. "\
+        log_warn "invalid internal_locale #{internal_locale.inspect}. " \
                  "Available internal locales: #{valid_locales * ', '}."
         internal_locale = DEFAULTS[:internal_locale].to_s
       end

data/lib/i18n/tasks/data/adapter/yaml_adapter.rb

@@ -6,6 +6,7 @@ module I18n::Tasks
     module Adapter
       module YamlAdapter
         EMOJI_REGEX = /\\u[\da-f]{8}/i.freeze
+        TRAILING_SPACE_REGEX = / $/.freeze
 
         class << self
           # @return [Hash] locale tree
@@ -20,13 +21,18 @@ module I18n::Tasks
 
           # @return [String]
           def dump(tree, options)
-            restore_emojis(tree.to_yaml(options || {}))
+            strip_trailing_spaces(restore_emojis(tree.to_yaml(options || {})))
           end
 
           # @return [String]
           def restore_emojis(yaml)
             yaml.gsub(EMOJI_REGEX) { |m| [m[-8..].to_i(16)].pack('U') }
           end
+
+          # @return [String]
+          def strip_trailing_spaces(yaml)
+            yaml.gsub(TRAILING_SPACE_REGEX, '')
+          end
         end
       end
     end

data/lib/i18n/tasks/data/tree/traversal.rb

@@ -161,12 +161,12 @@ module I18n::Tasks
       end
 
       def grep_keys(match, opts = {})
-        select_keys(opts) do |full_key, _node|
+        select_keys(**opts) do |full_key, _node|
           match === full_key # rubocop:disable Style/CaseEquality
         end
       end
 
-      def set_each_value!(val_pattern, key_pattern = nil, &value_proc)
+      def set_each_value!(val_pattern, key_pattern = nil, &value_proc) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         value_proc ||= proc do |node|
           node_value = node.value
           next node_value if node.reference?
@@ -174,15 +174,29 @@ module I18n::Tasks
           human_key = ActiveSupport::Inflector.humanize(node.key.to_s)
           full_key = node.full_key
           default = (node.data[:occurrences] || []).detect { |o| o.default_arg.presence }.try(:default_arg)
-          StringInterpolation.interpolate_soft(
-            val_pattern,
-            value: node_value,
-            human_key: human_key,
-            key: full_key,
-            default: default,
-            value_or_human_key: node_value.presence || human_key,
-            value_or_default_or_human_key: node_value.presence || default || human_key
-          )
+          if default.is_a?(Hash)
+            default.each_with_object({}) do |(k, v), h|
+              h[k] = StringInterpolation.interpolate_soft(
+                val_pattern,
+                value: node_value,
+                human_key: human_key,
+                key: full_key,
+                default: v,
+                value_or_human_key: node_value.presence || human_key,
+                value_or_default_or_human_key: node_value.presence || v || human_key
+              )
+            end
+          else
+            StringInterpolation.interpolate_soft(
+              val_pattern,
+              value: node_value,
+              human_key: human_key,
+              key: full_key,
+              default: default,
+              value_or_human_key: node_value.presence || human_key,
+              value_or_default_or_human_key: node_value.presence || default || human_key
+            )
+          end
         end
         pattern_re = I18n::Tasks::KeyPatternMatching.compile_key_pattern(key_pattern) if key_pattern.present?
         keys.each do |key, node|

data/lib/i18n/tasks/key_pattern_matching.rb

@@ -21,6 +21,7 @@ module I18n::Tasks::KeyPatternMatching
   # In patterns:
   #      *     is like .* in regexs
   #      :     matches a single key
+  #      *:    matches part of a single key, equivalent to `[^.]+?` regex
   #   { a, b.c } match any in set, can use : and *, match is captured
   def compile_key_pattern(key_pattern)
     return key_pattern if key_pattern.is_a?(Regexp)
@@ -31,6 +32,7 @@ module I18n::Tasks::KeyPatternMatching
   def key_pattern_re_body(key_pattern)
     key_pattern
       .gsub(/\./, '\.')
+      .gsub(/\*:/, '[^.]+?')
       .gsub(/\*/, '.*')
       .gsub(/:/, '(?<=^|\.)[^.]+?(?=\.|$)')
       .gsub(/\{(.*?)}/) { "(#{Regexp.last_match(1).strip.gsub(/\s*,\s*/, '|')})" }

data/lib/i18n/tasks/locale_pathname.rb

@@ -10,7 +10,7 @@ module I18n::Tasks
       private
 
       def path_locale_re(locale)
-        (@path_locale_res ||= {})[locale] ||= %r{(?<=^|[/.])#{locale}(?=[/.])}
+        (@path_locale_res ||= {})[locale] ||= %r{(?<=^|[/.-])#{locale}(?=[/.])}
       end
     end
   end

data/lib/i18n/tasks/plural_keys.rb

@@ -52,15 +52,9 @@ module I18n::Tasks::PluralKeys
   end
 
   def plural_forms?(s)
-    return false if non_plural_other?(s)
-
     s.present? && s.all? { |node| node.leaf? && plural_suffix?(node.key) }
   end
 
-  def non_plural_other?(s)
-    s.size == 1 && s.first.leaf? && (!s.first.value.is_a?(String) || !s.first.value.include?('%{count}'))
-  end
-
   def plural_suffix?(key)
     PLURAL_KEY_SUFFIXES.include?(key)
   end

data/lib/i18n/tasks/references.rb

@@ -90,9 +90,9 @@ module I18n::Tasks
           on_leaves_merge: lambda do |node, other|
             if node.value != other.value
               log_warn(
-                'Conflicting references: '\
-                "#{node.full_key(root: false)} ⮕ #{node.value} in #{node.data[:locale]},"\
-                " but ⮕ #{other.value} in #{other.data[:locale]}"
+                'Conflicting references: ' \
+                "#{node.full_key(root: false)} ⮕ #{node.value} in #{node.data[:locale]}, " \
+                "but ⮕ #{other.value} in #{other.data[:locale]}"
               )
             end
           end

data/lib/i18n/tasks/reports/base.rb

@@ -36,7 +36,7 @@ module I18n::Tasks::Reports
 
     def used_title(keys_nodes, filter)
       used_n = keys_nodes.map { |_k, node| node.data[:occurrences].size }.reduce(:+).to_i
-      "#{keys_nodes.size} key#{'s' if keys_nodes.size != 1}#{" matching '#{filter}'" if filter}"\
+      "#{keys_nodes.size} key#{'s' if keys_nodes.size != 1}#{" matching '#{filter}'" if filter}" \
         "#{" (#{used_n} usage#{'s' if used_n != 1})" if used_n.positive?}"
     end
 

data/lib/i18n/tasks/reports/terminal.rb

@@ -112,7 +112,7 @@ module I18n
           when :missing_plural
             leaf[:data][:missing_keys].join(', ')
           else
-            "#{Rainbow(leaf[:data][:missing_diff_locale]).cyan} "\
+            "#{Rainbow(leaf[:data][:missing_diff_locale]).cyan} " \
             "#{format_value(leaf[:value].is_a?(String) ? leaf[:value].strip : leaf[:value])}"
           end
         end

data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb

@@ -54,6 +54,22 @@ module I18n::Tasks::Scanners::AstMatchers
       end
     end
 
+    # Extract the whole hash from a node of type `:hash`
+    #
+    # @param node [AST::Node] a node of type `:hash`.
+    # @return [Hash] the whole hash from the node
+    def extract_hash(node)
+      return {} if node.nil?
+
+      if node.type == :hash
+        node.children.each_with_object({}) do |pair, h|
+          key = pair.children[0].children[0].to_s
+          value = pair.children[1].children[0]
+          h[key] = value
+        end
+      end
+    end
+
     # Extract a hash pair with a given literal key.
     #
     # @param node [AST::Node] a node of type `:hash`.

data/lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb

@@ -77,8 +77,13 @@ module I18n::Tasks::Scanners::AstMatchers
 
         key = [scope, key].join('.') unless scope == ''
       end
-      default_arg_node = extract_hash_pair(node, 'default')
-      default_arg = extract_string(default_arg_node.children[1]) if default_arg_node
+      if default_arg_node = extract_hash_pair(node, 'default')
+        default_arg = if default_arg_node.children[1]&.type == :hash
+                        extract_hash(default_arg_node.children[1])
+                      else
+                        extract_string(default_arg_node.children[1])
+                      end
+      end
 
       [key, default_arg]
     end

data/lib/i18n/tasks/scanners/pattern_scanner.rb

@@ -12,7 +12,7 @@ module I18n::Tasks::Scanners
     include OccurrenceFromPosition
     include RubyKeyLiterals
 
-    TRANSLATE_CALL_RE = /(?<=^|[^\w'\-.]|[^\w'\-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
+    TRANSLATE_CALL_RE = /(?<=^|[^\w'\-.]|[^\w'-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
     IGNORE_LINES = {
       'coffee' => /^\s*#(?!\si18n-tasks-use)/,
       'erb' => /^\s*<%\s*#(?!\si18n-tasks-use)/,

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -4,6 +4,7 @@ require 'i18n/tasks/scanners/file_scanner'
 require 'i18n/tasks/scanners/relative_keys'
 require 'i18n/tasks/scanners/ruby_ast_call_finder'
 require 'i18n/tasks/scanners/ast_matchers/message_receivers_matcher'
+require 'i18n/tasks/scanners/ast_matchers/rails_model_matcher'
 require 'parser/current'
 
 module I18n::Tasks::Scanners

data/lib/i18n/tasks/translation.rb

@@ -2,13 +2,14 @@
 
 require 'i18n/tasks/translators/deepl_translator'
 require 'i18n/tasks/translators/google_translator'
+require 'i18n/tasks/translators/openai_translator'
 require 'i18n/tasks/translators/yandex_translator'
 
 module I18n::Tasks
   module Translation
     # @param [I18n::Tasks::Tree::Siblings] forest to translate to the locales of its root nodes
     # @param [String] from locale
-    # @param [:deepl, :google, :yandex] backend
+    # @param [:deepl, :openai, :google, :yandex] backend
     # @return [I18n::Tasks::Tree::Siblings] translated forest
     def translate_forest(forest, from:, backend: :google)
       case backend
@@ -16,6 +17,8 @@ module I18n::Tasks
         Translators::DeeplTranslator.new(self).translate_forest(forest, from)
       when :google
         Translators::GoogleTranslator.new(self).translate_forest(forest, from)
+      when :openai
+        Translators::OpenAiTranslator.new(self).translate_forest(forest, from)
       when :yandex
         Translators::YandexTranslator.new(self).translate_forest(forest, from)
       else

data/lib/i18n/tasks/translators/base_translator.rb

@@ -3,6 +3,7 @@
 module I18n::Tasks
   module Translators
     class BaseTranslator
+      include ::I18n::Tasks::Logging
       # @param [I18n::Tasks::BaseTask] i18n_tasks
       def initialize(i18n_tasks)
         @i18n_tasks = i18n_tasks
@@ -31,7 +32,7 @@ module I18n::Tasks
         reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
         list -= reference_key_vals
         result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
-          fetch_translations list_slice, opts.merge(is_html ? options_for_html : options_for_plain)
+          fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
         end.reduce(:+) || []
         result.concat(reference_key_vals)
         result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
@@ -41,34 +42,36 @@ module I18n::Tasks
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<[String, Object]>] translated list
       def fetch_translations(list, opts)
-        from_values(list, translate_values(to_values(list), **options_for_translate_values(**opts))).tap do |result|
+        options = options_for_translate_values(**opts)
+        from_values(list, translate_values(to_values(list, options), **options), options).tap do |result|
           fail CommandError, no_results_error_message if result.blank?
         end
       end
 
       # @param [Array<[String, Object]>] list of key-value pairs
       # @return [Array<String>] values for translation extracted from list
-      def to_values(list)
-        list.map { |l| dump_value l[1] }.flatten.compact
+      def to_values(list, opts)
+        list.map { |l| dump_value(l[1], opts) }.flatten.compact
       end
 
       # @param [Array<[String, Object]>] list
       # @param [Array<String>] translated_values
       # @return [Array<[String, Object]>] translated key-value pairs
-      def from_values(list, translated_values)
+      def from_values(list, translated_values, opts)
         keys = list.map(&:first)
         untranslated_values = list.map(&:last)
-        keys.zip parse_value(untranslated_values, translated_values.to_enum)
+        keys.zip parse_value(untranslated_values, translated_values.to_enum, opts)
       end
 
       # Prepare value for translation.
       # @return [String, Array<String, nil>, nil] value for Google Translate or nil for non-string values
-      def dump_value(value)
+      def dump_value(value, opts)
         case value
         when Array
           # dump recursively
-          value.map { |v| dump_value v }
+          value.map { |v| dump_value(v, opts) }
         when String
+          value = CGI.escapeHTML(value) if opts[:html_escape]
           replace_interpolations value unless value.empty?
         end
       end
@@ -77,16 +80,18 @@ module I18n::Tasks
       # @param [Object] untranslated
       # @param [Enumerator] each_translated
       # @return [Object] final translated value
-      def parse_value(untranslated, each_translated)
+      def parse_value(untranslated, each_translated, opts)
         case untranslated
         when Array
           # implode array
-          untranslated.map { |from| parse_value(from, each_translated) }
+          untranslated.map { |from| parse_value(from, each_translated, opts) }
         when String
           if untranslated.empty?
             untranslated
           else
-            restore_interpolations untranslated, each_translated.next
+            value = each_translated.next
+            value = CGI.unescapeHTML(value) if opts[:html_escape]
+            restore_interpolations(untranslated, value)
           end
         else
           untranslated

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -4,6 +4,11 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class DeeplTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    # those languages must be specified with their sub-kind e.g en-us
+    SPECIFIC_TARGETS = %w[en pt].freeze
+
     def initialize(*)
       begin
         require 'deepl'
@@ -17,16 +22,22 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, from:, to:, **options)
-      result = DeepL.translate(list, to_deepl_compatible_locale(from), to_deepl_compatible_locale(to), options)
-      if result.is_a?(DeepL::Resources::Text)
-        [result.text]
-      else
-        result.map(&:text)
+      results = []
+      list.each_slice(BATCH_SIZE) do |parts|
+        res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options)
+        if res.is_a?(DeepL::Resources::Text)
+          results << res.text
+        else
+          results += res.map(&:text)
+        end
       end
+      results
     end
 
     def options_for_translate_values(**options)
-      { ignore_tags: %w[i18n] }.merge(options)
+      extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
+
+      extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
     end
 
     def options_for_html
@@ -34,7 +45,7 @@ module I18n::Tasks::Translators
     end
 
     def options_for_plain
-      { preserve_formatting: true }
+      { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
     end
 
     # @param [String] value
@@ -60,11 +71,23 @@ module I18n::Tasks::Translators
 
     private
 
-    # Convert 'es-ES' to 'ES'
-    def to_deepl_compatible_locale(locale)
+    # Convert 'es-ES' to 'ES', en-us to EN
+    def to_deepl_source_locale(locale)
       locale.to_s.split('-', 2).first.upcase
     end
 
+    # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
+    def to_deepl_target_locale(locale)
+      loc, sub = locale.to_s.split('-')
+      if SPECIFIC_TARGETS.include?(loc)
+        # Must see how the deepl api evolves, so this could be an error in the future
+        warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
+        locale.to_s.upcase
+      else
+        loc.upcase
+      end
+    end
+
     def configure_api_key!
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]

data/lib/i18n/tasks/translators/google_translator.rb

@@ -55,7 +55,7 @@ module I18n::Tasks::Translators
         key = @i18n_tasks.translation_config[:google_translate_api_key]
         # fallback with deprecation warning
         if @i18n_tasks.translation_config[:api_key]
-          @i18n_tasks.warn_deprecated(
+          warn_deprecated(
             'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
           )
           key ||= translation_config[:api_key]

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/translators/base_translator'
+
+module I18n::Tasks::Translators
+  class OpenAiTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+
+    def initialize(*)
+      begin
+        require 'openai'
+      rescue LoadError
+        raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command"
+      end
+      super
+    end
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t('i18n_tasks.openai_translate.errors.no_results')
+    end
+
+    private
+
+    def translator
+      @translator ||= OpenAI::Client.new(access_token: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:openai_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank?
+
+        key
+      end
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+
+        results << JSON.parse(translations)
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to) # rubocop:disable Metrics/MethodLength
+      messages = [
+        {
+          role: 'system',
+          content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n
+          locale array. The array has a structured format and contains multiple strings. Your task is to translate
+          each of these strings and create a new array with the translated strings. Keep in mind the context of all
+          the strings for a more accurate translation.\n"
+        },
+        {
+          role: 'user',
+          content: "Translate this array: \n\n\n"
+        },
+        {
+          role: 'user',
+          content: values.to_json
+        }
+      ]
+
+      response = translator.chat(
+        parameters: {
+          model: 'gpt-3.5-turbo',
+          messages: messages,
+          temperature: 0.7
+        }
+      )
+
+      translations = response.dig('choices', 0, 'message', 'content')
+      error = response['error']
+
+      fail "AI error: #{error}" if error.present?
+
+      translations
+    end
+  end
+end

data/lib/i18n/tasks/used_keys.rb

@@ -28,8 +28,8 @@ module I18n::Tasks
       strict: true
     }.freeze
 
-    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss *.less
-                        *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
+    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss
+                        *.less *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
                         *.webp *.map *.xlsx].freeze
 
     # Find all keys in the source and return a forest with the keys in absolute form and their occurrences.
@@ -142,7 +142,7 @@ module I18n::Tasks
 
     # keys in the source that end with a ., e.g. t("category.#{ cat.i18n_key }") or t("category." + category.key)
     # @param [String] replacement for interpolated values.
-    def expr_key_re(replacement: ':')
+    def expr_key_re(replacement: '*:')
       @expr_key_re ||= begin
         # disallow patterns with no keys
         ignore_pattern_re = /\A[.#{replacement}]*\z/

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '1.0.11'
+    VERSION = '1.0.13'
   end
 end

data/templates/config/i18n-tasks.yml

@@ -13,7 +13,7 @@ data:
   ## Provide a custom adapter:
   # adapter: I18n::Tasks::Data::FileSystem
 
-  # Locale files or `File.find` patterns where translations are read from:
+  # Locale files or `Find.find` patterns where translations are read from:
   read:
     ## Default:
     # - config/locales/%{locale}.yml
@@ -52,7 +52,7 @@ data:
 
 # Find translate calls
 search:
-  ## Paths or `File.find` patterns to search in:
+  ## Paths or `Find.find` patterns to search in:
   # paths:
   #  - app/
 
@@ -94,7 +94,7 @@ search:
   ##     User.human_attribute_name(:email) and User.model_name.human
   ##
   ## To implement your own, please see `I18n::Tasks::Scanners::AstMatchers::BaseMatcher`.
-  <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
+  # <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
 
   ## Multiple scanners can be used. Their results are merged.
   ## The options specified above are passed down to each scanner. Per-scanner options can be specified as well.
@@ -110,7 +110,9 @@ search:
 #   deepl_api_key: "48E92789-57A3-466A-9959-1A1A1A1A1A1A"
 #   # deepl_host: "https://api.deepl.com"
 #   # deepl_version: "v2"
-
+#   # add additional options to the DeepL.translate call: https://www.deepl.com/docs-api/translate-text/translate-text/
+#   deepl_options:
+#     formality: prefer_less
 ## Do not consider these keys missing:
 # ignore_missing:
 # - 'errors.messages.{accepted,blank,invalid,too_short,too_long}'

data/templates/minitest/i18n_test.rb

@@ -5,18 +5,18 @@ require 'i18n/tasks'
 class I18nTest < ActiveSupport::TestCase
   def setup
     @i18n = I18n::Tasks::BaseTask.new
-    @missing_keys = @i18n.missing_keys
-    @unused_keys = @i18n.unused_keys
   end
 
   def test_no_missing_keys
-    assert_empty @missing_keys,
-                 "Missing #{@missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
+    missing_keys = @i18n.missing_keys
+    assert_empty missing_keys,
+                 "Missing #{missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
   end
 
   def test_no_unused_keys
-    assert_empty @unused_keys,
-                 "#{@unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
+    unused_keys = @i18n.unused_keys
+    assert_empty unused_keys,
+                 "#{unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
   end
 
   def test_files_are_normalized

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.13
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-15 00:00:00.000000000 Z
+date: 2023-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -42,16 +42,22 @@ dependencies:
   name: better_html
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +106,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
 - !ruby/object:Gem::Dependency
   name: rails-i18n
   requirement: !ruby/object:Gem::Requirement
@@ -224,14 +230,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.27.0
+        version: 1.50.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.50.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.27.0
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -402,6 +436,7 @@ files:
 - lib/i18n/tasks/translators/base_translator.rb
 - lib/i18n/tasks/translators/deepl_translator.rb
 - lib/i18n/tasks/translators/google_translator.rb
+- lib/i18n/tasks/translators/openai_translator.rb
 - lib/i18n/tasks/translators/yandex_translator.rb
 - lib/i18n/tasks/unused_keys.rb
 - lib/i18n/tasks/used_keys.rb