i18n-tasks 1.0.13 → 1.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12e19e4d7fe61bd20e000b0b6e4db5bdf233443e8e8cf965b62e634bdfa3e5a2
-  data.tar.gz: 3dab05338d079c4defad590466ab2985faf43245d616f86af4c212003831a6e4
+  metadata.gz: bd61ceb254dc1c44768bd40672b3f31635c9d42fb237950cbdfe64fef642a850
+  data.tar.gz: ae6981dd739636ddb5f1bae1ca9e5cf05ba991fd1345a5488e1cfa8561833877
 SHA512:
-  metadata.gz: 2cf99aec1c6d5738c4a71964a7ce67e70c8ade3150042c4e381fef19b9eaa82cf6638979bf97c6e9941f119304973f9e6f3fd31b3bab0fd635520344448b5587
-  data.tar.gz: 2de5ab47951585b298e43bece202f3ad885ae8eb03edb2b6e860b30e3172ea213947509b9543bbf31df06ef03079ff1e7cfd1b7a32cd964428d9f9612fd7e171
+  metadata.gz: 1312910139411cc4695bea0f0de8624d49958efac33c8c7b7bd99fb3f351d408d0a6980789ec81a43452bc29a15292f6de9719ace0e3b902866d63c02f5cab79
+  data.tar.gz: e782f1bc1f1ad68cc33583d014c879b3d121ba2e0990db8cc109f864aeb0e98f0a97030300294e31e9041899b131b2b677fe6917f705d09bda6bf07aed588b42

data/README.md

@@ -24,7 +24,7 @@ i18n-tasks can be used with any project using the ruby [i18n gem][i18n-gem] (def
 Add i18n-tasks to the Gemfile:
 
 ```ruby
-gem 'i18n-tasks', '~> 1.0.13'
+gem 'i18n-tasks', '~> 1.0.14', group: :development
 ```
 
 Copy the default [configuration file](#configuration):
@@ -338,6 +338,33 @@ data:
 If you want to have i18n-tasks reorganize your existing keys using `data.write`, either set the router to
 `pattern_router` as above, or run `i18n-tasks normalize -p` (forcing the use of the pattern router for that run).
 
+##### Isolating router
+
+Isolating router assumes each YAML file is independent and can contain similar keys.
+
+As a result, the translations are written to an alternate target file for each source file
+(only the `%{locale}` part is changed to match target locale). Thus, it is not necessary to
+specify any `write` configuration (in fact, it would be completely ignored).
+
+This can be useful for example when using [ViewComponent sidecars](https://viewcomponent.org/guide/translations.html)
+(ViewComponent assigns an implicit scope to each sidecar YAML file but `i18n-tasks` is not aware of
+that logic, resulting in collisions):
+
+* `app/components/movies_component.en.yml`:
+   ```yaml
+   en:
+     title: Movies
+   ```
+
+* `app/components/games_component.en.yml`
+   ```yaml
+   en:
+     title: Games
+   ```
+
+This router has a limitation, though: it does not support detecting missing keys from code usage
+(since it is not aware of the implicit scope logic).
+
 ##### Key pattern syntax
 
 A special syntax similar to file glob patterns is used throughout i18n-tasks to match translation keys:
@@ -411,6 +438,12 @@ translation:
   google_translate_api_key: <Google Translate API key>
 ```
 
+or via environment variable:
+
+```bash
+GOOGLE_TRANSLATE_API_KEY=<Google Translate API key>
+```
+
 <a name="deepl-translation-config"></a>
 ### DeepL Pro Translate
 
@@ -422,6 +455,19 @@ translation:
   deepl_api_key: <DeepL Pro API key>
   deepl_host: <optional>
   deepl_version: <optional>
+  deepl_glossary_ids:
+    - f28106eb-0e06-489e-82c6-8215d6f95089
+    - 2c6415be-1852-4f54-9e1b-d800463496b4
+  deepl_options:
+    formality: prefer_less
+```
+
+or via environment variables:
+
+```bash
+DEEPL_API_KEY=<DeepL Pro API key>
+DEEPL_HOST=<optional>
+DEEPL_VERSION=<optional>
 ```
 
 <a name="yandex-translation-config"></a>
@@ -435,6 +481,12 @@ translation:
   yandex_api_key: <Yandex API key>
 ```
 
+or via environment variable:
+
+```bash
+YANDEX_API_KEY=<Yandex API key>
+```
+
 <a name="openai-translation-config"></a>
 ### OpenAI Translate
 
@@ -444,6 +496,14 @@ translation:
 # config/i18n-tasks.yml
 translation:
   openai_api_key: <OpenAI API key>
+  openai_model: <optional>
+```
+
+or via environment variable:
+
+```bash
+OPENAI_API_KEY=<OpenAI API key>
+OPENAI_MODEL=<optional>
 ```
 
 ## Interactive console

data/config/locales/en.yml

@@ -16,12 +16,10 @@ en:
           data_format: 'Data format: %{valid_text}.'
           keep_order: Keep the order of the keys
           key_pattern: Filter by key pattern (e.g. 'common.*')
-          key_pattern_to_rename: Full key (pattern) to rename. Required
           locale: :i18n_tasks.common.locale
           locale_to_translate_from: Locale to translate from
           locales_filter: 'Locale(s) to process. Special: base'
           missing_types: 'Filter by types: %{valid}'
-          new_key_name: New name, interpolates original name as %{key}. Required
           nostdin: Do not read from stdin
           out_format: 'Output format: %{valid_text}'
           pattern_router: 'Use pattern router: keys moved per config data.write'
@@ -30,13 +28,14 @@ en:
             the config setting if set.
           translation_backend: Translation backend (google or deepl)
           value: >-
-            Value. Interpolates: %{value}, %{human_key}, %{key}, %{default}, %{value_or_human_key},
-            %{value_or_default_or_human_key}
+            Value. Interpolates: %%{value}, %%{human_key}, %%{key}, %%{default}, %%{value_or_human_key},
+            %%{value_or_default_or_human_key}
       desc:
         add_missing: add missing keys to locale data, optionally match a pattern
         check_consistent_interpolations: verify that all translations use correct interpolation variables
         check_normalized: verify that all translation data is normalized
         config: display i18n-tasks configuration
+        cp: copy the keys in locale data that match the given pattern
         data: show locale data
         data_merge: merge locale data with trees
         data_remove: remove keys present in tree from data

data/config/locales/ru.yml

@@ -13,28 +13,27 @@ ru:
           data_format: 'Формат данных: %{valid_text}.'
           keep_order: Keep the order of the keys
           key_pattern: Маска ключа (например, common.*)
-          key_pattern_to_rename: Полный ключ (шаблон) для переименования. Необходимый параметр.
           locale: 'Язык. По умолчанию: base'
           locale_to_translate_from: 'Язык, с которого переводить (по умолчанию: base)'
           locales_filter: >-
             Список языков для обработки, разделенный запятыми (,). По умолчанию: все. Специальное
             значение: base.
           missing_types: 'Типы недостающих переводов: %{valid}. По умолчанию: все'
-          new_key_name: Новое имя, интерполирует оригинальное название как %{key}. Необходимый параметр.
           nostdin: Не читать дерево из стандартного ввода
           out_format: 'Формат вывода: %{valid_text}.'
           pattern_router: 'Использовать pattern_router: ключи распределятся по файлам согласно data.write'
           strict: Не угадывать динамические использования ключей, например `t("category.#{category.key}")`
           translation_backend: Движок перевода (google или deepl)
           value: >-
-            Значение, интерполируется с %{value}, %{human_key}, %{key}, %{default}, %{value_or_human_key},
-            %{value_or_default_or_human_key}
+            Значение, интерполируется с %%{value}, %%{human_key}, %%{key}, %%{default}, %%{value_or_human_key},
+            %%{value_or_default_or_human_key}
       desc:
         add_missing: добавить недостающие ключи к переводам
         check_consistent_interpolations: убедитесь, что во всех переводах используются правильные
           интерполяционные переменные
         check_normalized: проверить, что все файлы переводов нормализованы
         config: показать конфигурацию
+        cp: скопируйте ключи в данных локали, соответствующие заданному шаблону
         data: показать данные переводов
         data_merge: добавить дерево к переводам
         data_remove: удалить ключи, которые есть в дереве, из данных

data/i18n-tasks.gemspec

@@ -39,7 +39,6 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'activesupport', '>= 4.0.2'
   s.add_dependency 'ast', '>= 2.1.0'
-  s.add_dependency 'better_html', '>= 1.0', '< 3.0'
   s.add_dependency 'erubi'
   s.add_dependency 'highline', '>= 2.0.0'
   s.add_dependency 'i18n'

data/lib/i18n/tasks/command/commands/data.rb

@@ -46,6 +46,20 @@ module I18n::Tasks
           terminal_report.mv_results results
         end
 
+        cmd :cp,
+            pos: 'FROM_KEY_PATTERN TO_KEY_PATTERN',
+            desc: t('i18n_tasks.cmd.desc.cp')
+        def cp(opt = {})
+          fail CommandError, 'requires FROM_KEY_PATTERN and TO_KEY_PATTERN' if opt[:arguments].size < 2
+
+          from_pattern = opt[:arguments].shift
+          to_pattern = opt[:arguments].shift
+          forest = i18n.data_forest
+          results = forest.mv_key!(compile_key_pattern(from_pattern), to_pattern, root: false, retain: true)
+          i18n.data.write forest
+          terminal_report.cp_results results
+        end
+
         cmd :rm,
             pos: 'KEY_PATTERN [KEY_PATTERN...]',
             desc: t('i18n_tasks.cmd.desc.rm')

data/lib/i18n/tasks/command/options/common.rb

@@ -26,7 +26,7 @@ module I18n::Tasks
         arg :value,
             '-v',
             '--value VALUE',
-            t('i18n_tasks.cmd.args.desc.value')
+            t('i18n_tasks.cmd.args.desc.value', dummy: 'value') # Dummy value is workaround for https://github.com/ruby-i18n/i18n/issues/689
 
         arg :config,
             '-c',

data/lib/i18n/tasks/configuration.rb

@@ -67,6 +67,7 @@ module I18n::Tasks::Configuration # rubocop:disable Metrics/ModuleLength
       conf[:deepl_host] = ENV['DEEPL_HOST'] if ENV.key?('DEEPL_HOST')
       conf[:deepl_version] = ENV['DEEPL_VERSION'] if ENV.key?('DEEPL_VERSION')
       conf[:openai_api_key] = ENV['OPENAI_API_KEY'] if ENV.key?('OPENAI_API_KEY')
+      conf[:openai_model] = ENV['OPENAI_MODEL'] if ENV.key?('OPENAI_MODEL')
       conf[:yandex_api_key] = ENV['YANDEX_API_KEY'] if ENV.key?('YANDEX_API_KEY')
       conf
     end

data/lib/i18n/tasks/data/file_system_base.rb

@@ -3,6 +3,7 @@
 require 'i18n/tasks/data/tree/node'
 require 'i18n/tasks/data/router/pattern_router'
 require 'i18n/tasks/data/router/conservative_router'
+require 'i18n/tasks/data/router/isolating_router'
 require 'i18n/tasks/data/file_formats'
 require 'i18n/tasks/key_pattern_matching'
 
@@ -151,6 +152,7 @@ module I18n::Tasks
 
       ROUTER_NAME_ALIASES = {
         'conservative_router' => 'I18n::Tasks::Data::Router::ConservativeRouter',
+        'isolating_router' => 'I18n::Tasks::Data::Router::IsolatingRouter',
         'pattern_router' => 'I18n::Tasks::Data::Router::PatternRouter'
       }.freeze
       def router
@@ -170,6 +172,9 @@ module I18n::Tasks
         end.map do |path|
           [path.freeze, load_file(path) || {}]
         end.map do |path, data|
+          if router.is_a?(I18n::Tasks::Data::Router::IsolatingRouter)
+            data.transform_values! { |tree| { "<#{router.alternate_path_for(path, base_locale)}>" => tree } }
+          end
           filter_nil_keys! path, data
           Data::Tree::Siblings.from_nested_hash(data).tap do |s|
             s.leaves { |x| x.data.update(path: path, locale: locale) }

data/lib/i18n/tasks/data/router/isolating_router.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/key_pattern_matching'
+require 'i18n/tasks/data/tree/node'
+
+module I18n::Tasks
+  module Data::Router
+    # Route based on source file path
+    class IsolatingRouter
+      include ::I18n::Tasks::KeyPatternMatching
+
+      attr_reader :config_read_patterns, :base_locale
+
+      def initialize(_adapter, data_config)
+        @base_locale = data_config[:base_locale]
+        @config_read_patterns = Array.wrap(data_config[:read])
+      end
+
+      # Route keys to destinations
+      # @param forest [I18n::Tasks::Data::Tree::Siblings] forest roots are locales.
+      # @yieldparam [String] dest_path
+      # @yieldparam [I18n::Tasks::Data::Tree::Siblings] tree_slice
+      # @return [Hash] mapping of destination => [ [key, value], ... ]
+      def route(locale, forest, &block)
+        return to_enum(:route, locale, forest) unless block
+
+        locale = locale.to_s
+        out = {}
+
+        forest.keys do |key_namespaced_with_source_path, _node|
+          source_path, key = key_namespaced_with_source_path.match(/\A<([^>]*)>\.(.*)/).captures
+          target_path = alternate_path_for(source_path, locale)
+          next unless source_path && key && target_path
+
+          (out[target_path] ||= Set.new) << "#{locale}.#{key}"
+        end
+
+        out.each do |target_path, keys|
+          file_namespace_subtree = I18n::Tasks::Data::Tree::Siblings.new(
+            nodes: forest.get("#{locale}.<#{alternate_path_for(target_path, base_locale)}>")
+          )
+          file_namespace_subtree.set_root_key!(locale)
+
+          block.yield(
+            target_path,
+            file_namespace_subtree.select_keys { |key, _| keys.include?(key) }
+          )
+        end
+      end
+
+      def alternate_path_for(source_path, locale)
+        source_path = source_path.dup
+
+        config_read_patterns.each do |pattern|
+          regexp = Glob.new(format(pattern, locale: '(*)')).to_regexp
+          next unless source_path.match?(regexp)
+
+          source_path.match(regexp) do |match_data|
+            (1..match_data.size - 1).reverse_each do |capture_index|
+              capture_begin, capture_end = match_data.offset(capture_index)
+              source_path.slice!(Range.new(capture_begin, capture_end, true))
+              source_path.insert(capture_begin, locale.to_s)
+            end
+          end
+
+          return source_path
+        end
+
+        nil
+      end
+
+      # based on https://github.com/alexch/rerun/blob/36f2d237985b670752abbe4a7f6814893cdde96f/lib/rerun/glob.rb
+      class Glob
+        NO_LEADING_DOT = '(?=[^\.])'
+        START_OF_FILENAME = '(?:\A|\/)'
+        END_OF_STRING = '\z'
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def to_regexp_string # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
+          chars = smoosh(@pattern.chars)
+
+          curlies = 0
+          escaping = false
+
+          string = chars.map do |char|
+            if escaping
+              escaping = false
+              next char
+            end
+
+            case char
+            when '**' then '(?:[^/]+/)*'
+            when '*' then '.*'
+            when '?' then '.'
+            when '.' then '\.'
+            when '{'
+              curlies += 1
+              '('
+            when '}'
+              if curlies.positive?
+                curlies -= 1
+                ')'
+              else
+                char
+              end
+            when ','
+              if curlies.positive?
+                '|'
+              else
+                char
+              end
+            when '\\'
+              escaping = true
+              '\\'
+            else char
+            end
+          end.join
+
+          START_OF_FILENAME + string + END_OF_STRING
+        end
+
+        def to_regexp
+          Regexp.new(to_regexp_string)
+        end
+
+        def smoosh(chars)
+          out = []
+          until chars.empty?
+            char = chars.shift
+            if char == '*' && chars.first == '*'
+              chars.shift
+              chars.shift if chars.first == '/'
+              out.push('**')
+            else
+              out.push(char)
+            end
+          end
+          out
+        end
+      end
+    end
+  end
+end

data/lib/i18n/tasks/data/tree/siblings.rb

@@ -36,7 +36,7 @@ module I18n::Tasks::Data::Tree
     # @param to_pattern [Regexp]
     # @param root [Boolean]
     # @return {old key => new key}
-    def mv_key!(from_pattern, to_pattern, root: false) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def mv_key!(from_pattern, to_pattern, root: false, retain: false) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       moved_forest = Siblings.new
       moved_nodes = []
       old_key_to_new_key = {}
@@ -69,7 +69,7 @@ module I18n::Tasks::Data::Tree
           node.value = new_target.to_sym
         end
       end
-      remove_nodes_and_emptied_ancestors! moved_nodes
+      remove_nodes_and_emptied_ancestors!(moved_nodes) unless retain
       merge! moved_forest
       old_key_to_new_key
     end

data/lib/i18n/tasks/interpolations.rb

@@ -5,7 +5,7 @@ module I18n::Tasks
     class << self
       attr_accessor :variable_regex
     end
-    @variable_regex = /%{[^}]+}/.freeze
+    @variable_regex = /(?<!%)%{[^}]+}/.freeze
 
     def inconsistent_interpolations(locales: nil, base_locale: nil) # rubocop:disable Metrics/AbcSize
       locales ||= self.locales

data/lib/i18n/tasks/key_pattern_matching.rb

@@ -31,10 +31,10 @@ module I18n::Tasks::KeyPatternMatching
 
   def key_pattern_re_body(key_pattern)
     key_pattern
-      .gsub(/\./, '\.')
-      .gsub(/\*:/, '[^.]+?')
-      .gsub(/\*/, '.*')
-      .gsub(/:/, '(?<=^|\.)[^.]+?(?=\.|$)')
+      .gsub('.', '\.')
+      .gsub('*:', '[^.]+?')
+      .gsub('*', '.*')
+      .gsub(':', '(?<=^|\.)[^.]+?(?=\.|$)')
       .gsub(/\{(.*?)}/) { "(#{Regexp.last_match(1).strip.gsub(/\s*,\s*/, '|')})" }
   end
 end

data/lib/i18n/tasks/reports/terminal.rb

@@ -93,6 +93,12 @@ module I18n
           end
         end
 
+        def cp_results(results)
+          results.each do |(from, to)|
+            print_info "#{Rainbow(from).cyan} #{Rainbow('+').yellow.bright} #{Rainbow(to).green}"
+          end
+        end
+
         def check_normalized_results(non_normalized)
           if non_normalized.empty?
             print_success 'All data is normalized'

data/lib/i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/scanners/ast_matchers/base_matcher'
+require 'i18n/tasks/scanners/results/occurrence'
+
+module I18n::Tasks::Scanners::AstMatchers
+  class DefaultI18nSubjectMatcher < BaseMatcher
+    def convert_to_key_occurrences(send_node, method_name, location: send_node.loc)
+      children = Array(send_node&.children)
+      return unless children[1] == :default_i18n_subject
+
+      key = @scanner.absolute_key(
+        '.subject',
+        location.expression.source_buffer.name,
+        calling_method: method_name
+      )
+      [
+        key,
+        I18n::Tasks::Scanners::Results::Occurrence.from_range(
+          raw_key: key,
+          range: location.expression
+        )
+      ]
+    end
+  end
+end

data/lib/i18n/tasks/scanners/erb_ast_scanner.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 require 'i18n/tasks/scanners/ruby_ast_scanner'
-require 'i18n/tasks/scanners/erb_ast_processor'
-require 'better_html/errors'
-require 'better_html/parser'
+require 'i18n/tasks/scanners/local_ruby_parser'
 
 module I18n::Tasks::Scanners
   # Scan for I18n.translate calls in ERB-file better-html and ASTs
   class ErbAstScanner < RubyAstScanner
+    DEFAULT_REGEXP = /<%(={1,2}|-|\#|%)?(.*?)([-=])?%>/m.freeze
+
     def initialize(**args)
       super(**args)
-      @erb_ast_processor = ErbAstProcessor.new
+      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
     end
 
     private
@@ -20,29 +20,59 @@ module I18n::Tasks::Scanners
     # @param path Path to file to parse
     # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
     def path_to_ast_and_comments(path)
-      parser = BetterHtml::Parser.new(make_buffer(path))
-      ast = convert_better_html(parser.ast)
-      @erb_ast_processor.process_and_extract_comments(ast)
+      comments = []
+      buffer = make_buffer(path)
+
+      children = []
+      buffer
+        .source
+        .scan(DEFAULT_REGEXP) do |indicator, code, tailch, _rspace|
+          match = Regexp.last_match
+          character = indicator ? indicator[0] : nil
+
+          start = match.begin(0) + 2 + (character&.size || 0)
+          stop = match.end(0) - 2 - (tailch&.size || 0)
+
+          case character
+          when '=', nil, '-'
+            parsed, parsed_comments = handle_code(buffer, code, start, stop)
+            comments.concat(parsed_comments)
+            children << parsed unless parsed.nil?
+          when '#', '#-'
+            comments << handle_comment(buffer, start, stop)
+          end
+        end
+
+      [root_node(children, buffer), comments]
     end
 
-    # Convert BetterHtml nodes to Parser::AST::Node
-    #
-    # @param node BetterHtml::Parser::AST::Node
-    # @return Parser::AST::Node
-    def convert_better_html(node)
-      definition = Parser::Source::Map::Definition.new(
-        node.location.begin,
-        node.location.begin,
-        node.location.begin,
-        node.location.end
-      )
-      Parser::AST::Node.new(
-        node.type,
-        node.children.map { |child| child.is_a?(BetterHtml::AST::Node) ? convert_better_html(child) : child },
-        {
-          location: definition
-        }
-      )
+    def handle_code(buffer, code, start, stop)
+      range = ::Parser::Source::Range.new(buffer, start, stop)
+      location =
+        Parser::Source::Map::Definition.new(
+          range.begin,
+          range.begin,
+          range.begin,
+          range.end
+        )
+      @ruby_parser.parse(code, location: location)
+    end
+
+    def handle_comment(buffer, start, stop)
+      range = ::Parser::Source::Range.new(buffer, start, stop)
+      ::Parser::Source::Comment.new(range)
+    end
+
+    def root_node(children, buffer)
+      range = ::Parser::Source::Range.new(buffer, 0, buffer.source.size)
+      location =
+        Parser::Source::Map::Definition.new(
+          range.begin,
+          range.begin,
+          range.begin,
+          range.end
+        )
+      ::Parser::AST::Node.new(:erb, children, location: location)
     end
   end
 end

data/lib/i18n/tasks/scanners/pattern_scanner.rb

@@ -12,7 +12,7 @@ module I18n::Tasks::Scanners
     include OccurrenceFromPosition
     include RubyKeyLiterals
 
-    TRANSLATE_CALL_RE = /(?<=^|[^\w'\-.]|[^\w'-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
+    TRANSLATE_CALL_RE = /(?<=^|[^\p{L}'\-.]|[^\p{L}'-]I18n\.|I18n\.)t(?:!|ranslate!?)?/.freeze
     IGNORE_LINES = {
       'coffee' => /^\s*#(?!\si18n-tasks-use)/,
       'erb' => /^\s*<%\s*#(?!\si18n-tasks-use)/,

data/lib/i18n/tasks/scanners/relative_keys.rb

@@ -61,7 +61,7 @@ module I18n
             "#{file_key.sub(/_controller$/, '')}.#{calling_method}"
           else
             # Remove _ prefix from partials
-            file_key.gsub(/\._/, DOT)
+            file_key.gsub('._', DOT)
           end
         end
       end

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -3,6 +3,7 @@
 require 'i18n/tasks/scanners/file_scanner'
 require 'i18n/tasks/scanners/relative_keys'
 require 'i18n/tasks/scanners/ruby_ast_call_finder'
+require 'i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher'
 require 'i18n/tasks/scanners/ast_matchers/message_receivers_matcher'
 require 'i18n/tasks/scanners/ast_matchers/rails_model_matcher'
 require 'parser/current'
@@ -80,7 +81,7 @@ module I18n::Tasks::Scanners
         results = []
 
         # method_name is not available at this stage
-        calls.each do |send_node, _method_name|
+        calls.each do |(send_node, _method_name)|
           @matchers.each do |matcher|
             result = matcher.convert_to_key_occurrences(
               send_node,

data/lib/i18n/tasks/split_key.rb

@@ -5,6 +5,11 @@ module I18n
     module SplitKey
       module_function
 
+      PARENTHESIS_PAIRS = %w({} [] () <>).freeze
+      START_KEYS = PARENTHESIS_PAIRS.to_set { |pair| pair[0] }.freeze
+      END_KEYS = PARENTHESIS_PAIRS.to_h { |pair| [pair[0], pair[1]] }.freeze
+      private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS
+
       # split a key by dots (.)
       # dots inside braces or parenthesis are not split on
       #
@@ -12,61 +17,39 @@ module I18n
       # split_key 'a.#{b.c}' # => ['a', '#{b.c}']
       # split_key 'a.b.c', 2 # => ['a', 'b.c']
       def split_key(key, max = Float::INFINITY)
-        parts = []
-        pos   = 0
         return [key] if max == 1
 
-        key_parts(key) do |part|
-          parts << part
-          pos += part.length + 1
-          if parts.length + 1 >= max
-            parts << key[pos..] unless pos == key.length
-            break
+        parts = []
+        current_parenthesis_end_char = nil
+        part = ''
+        key.each_char.with_index do |char, index|
+          if current_parenthesis_end_char
+            part += char
+            current_parenthesis_end_char = nil if char == current_parenthesis_end_char
+          elsif START_KEYS.include?(char)
+            part += char
+            current_parenthesis_end_char = END_KEYS[char]
+          elsif char == '.'
+            parts << part
+            if parts.size + 1 == max
+              remaining = key[(index + 1)..]
+              parts << remaining unless remaining.empty?
+              return parts
+            end
+            part = ''
+          else
+            part += char
           end
         end
-        parts
-      end
-
-      def last_key_part(key)
-        last = nil
-        key_parts(key) { |part| last = part }
-        last
-      end
 
-      # yield each key part
-      # dots inside braces or parenthesis are not split on
-      def key_parts(key, &block)
-        return enum_for(:key_parts, key) unless block
+        return parts if part.empty?
 
-        nesting = PARENS
-        counts  = PARENS_ZEROS # dup'd later if key contains parenthesis
-        delim   = '.'
-        from    = to = 0
-        key.each_char do |char|
-          if char == delim && PARENS_ZEROS == counts
-            block.yield key[from...to]
-            from = to = (to + 1)
-          else
-            nest_i, nest_inc = nesting[char]
-            if nest_i
-              counts = counts.dup if counts.frozen?
-              counts[nest_i] += nest_inc
-            end
-            to += 1
-          end
-        end
-        block.yield(key[from...to]) if from < to && to <= key.length
-        true
+        current_parenthesis_end_char ? parts.concat(part.split('.')) : parts << part
       end
 
-      PARENS = %w({} [] ()).each_with_object({}) do |s, h|
-        i              = h.size / 2
-        h[s[0].freeze] = [i, 1].freeze
-        h[s[1].freeze] = [i, -1].freeze
-      end.freeze
-      PARENS_ZEROS = Array.new(PARENS.size, 0).freeze
-      private_constant :PARENS
-      private_constant :PARENS_ZEROS
+      def last_key_part(key)
+        split_key(key).last
+      end
     end
   end
 end

data/lib/i18n/tasks/translators/base_translator.rb

@@ -70,6 +70,9 @@ module I18n::Tasks
         when Array
           # dump recursively
           value.map { |v| dump_value(v, opts) }
+        when Hash
+          # dump recursively
+          value.values.map { |v| dump_value(v, opts) }
         when String
           value = CGI.escapeHTML(value) if opts[:html_escape]
           replace_interpolations value unless value.empty?
@@ -85,6 +88,9 @@ module I18n::Tasks
         when Array
           # implode array
           untranslated.map { |from| parse_value(from, each_translated, opts) }
+        when Hash
+          # implode hash
+          untranslated.transform_values { |value| parse_value(value, each_translated, opts) }
         when String
           if untranslated.empty?
             untranslated

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -24,7 +24,12 @@ module I18n::Tasks::Translators
     def translate_values(list, from:, to:, **options)
       results = []
       list.each_slice(BATCH_SIZE) do |parts|
-        res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options)
+        res = DeepL.translate(
+          parts,
+          to_deepl_source_locale(from),
+          to_deepl_target_locale(to),
+          options_with_glossary(options, from, to)
+        )
         if res.is_a?(DeepL::Resources::Text)
           results << res.text
         else
@@ -100,5 +105,26 @@ module I18n::Tasks::Translators
         config.version = version unless version.blank?
       end
     end
+
+    def options_with_glossary(options, from, to)
+      glossary = find_glossary(from, to)
+      glossary ? { glossary_id: glossary.id }.merge(options) : options
+    end
+
+    def all_ready_glossaries
+      @all_ready_glossaries ||= DeepL.glossaries.list
+    end
+
+    def find_glossary(from, to)
+      config_glossary_ids = @i18n_tasks.translation_config[:deepl_glossary_ids]
+      return unless config_glossary_ids
+
+      all_ready_glossaries.find do |glossary|
+        glossary.ready \
+          && glossary.source_lang == from \
+          && glossary.target_lang == to \
+          && config_glossary_ids.include?(glossary.id)
+      end
+    end
   end
 end

data/lib/i18n/tasks/translators/google_translator.rb

@@ -4,6 +4,7 @@ require 'i18n/tasks/translators/base_translator'
 
 module I18n::Tasks::Translators
   class GoogleTranslator < BaseTranslator
+    NEWLINE_PLACEHOLDER = '<br id=i18n />'
     def initialize(*)
       begin
         require 'easy_translate'
@@ -16,14 +17,21 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      EasyTranslate.translate(list, options)
+      restore_newlines(
+        EasyTranslate.translate(
+          replace_newlines_with_placeholder(list, options[:html]),
+          options,
+          format: :text
+        ),
+        options[:html]
+      )
     end
 
     def options_for_translate_values(from:, to:, **options)
       options.merge(
         api_key: api_key,
-        from: to_google_translate_compatible_locale(from),
-        to: to_google_translate_compatible_locale(to)
+        from: from,
+        to: to
       )
     end
 
@@ -41,15 +49,6 @@ module I18n::Tasks::Translators
 
     private
 
-    SUPPORTED_LOCALES_WITH_REGION = %w[zh-CN zh-TW].freeze
-
-    # Convert 'es-ES' to 'es'
-    def to_google_translate_compatible_locale(locale)
-      return locale unless locale.include?('-') && !SUPPORTED_LOCALES_WITH_REGION.include?(locale)
-
-      locale.split('-', 2).first
-    end
-
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:google_translate_api_key]
@@ -65,5 +64,21 @@ module I18n::Tasks::Translators
         key
       end
     end
+
+    def replace_newlines_with_placeholder(list, html)
+      return list unless html
+
+      list.map do |value|
+        value.gsub("\n", NEWLINE_PLACEHOLDER)
+      end
+    end
+
+    def restore_newlines(translations, html)
+      return translations unless html
+
+      translations.map do |translation|
+        translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n")
+      end
+    end
   end
 end

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -1,11 +1,24 @@
 # frozen_string_literal: true
 
 require 'i18n/tasks/translators/base_translator'
+require 'active_support/core_ext/string/filters'
 
 module I18n::Tasks::Translators
   class OpenAiTranslator < BaseTranslator
     # max allowed texts per request
     BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a professional translator that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+
+      Keep in mind the context of all the strings for a more accurate translation.
+    PROMPT
 
     def initialize(*)
       begin
@@ -50,6 +63,14 @@ module I18n::Tasks::Translators
       end
     end
 
+    def model
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-3.5-turbo'
+    end
+
+    def system_prompt
+      @system_prompt ||= @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    end
+
     def translate_values(list, from:, to:)
       results = []
 
@@ -62,14 +83,11 @@ module I18n::Tasks::Translators
       results.flatten
     end
 
-    def translate(values, from, to) # rubocop:disable Metrics/MethodLength
+    def translate(values, from, to)
       messages = [
         {
           role: 'system',
-          content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n
-          locale array. The array has a structured format and contains multiple strings. Your task is to translate
-          each of these strings and create a new array with the translated strings. Keep in mind the context of all
-          the strings for a more accurate translation.\n"
+          content: format(system_prompt, from: from, to: to)
         },
         {
           role: 'user',
@@ -83,9 +101,9 @@ module I18n::Tasks::Translators
 
       response = translator.chat(
         parameters: {
-          model: 'gpt-3.5-turbo',
+          model: model,
           messages: messages,
-          temperature: 0.7
+          temperature: 0.0
         }
       )
 

data/lib/i18n/tasks/used_keys.rb

@@ -21,8 +21,8 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.html.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.html.erb *.rb] }]
       ],
       ast_matchers: [],
       strict: true

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '1.0.13'
+    VERSION = '1.0.14'
   end
 end

data/templates/config/i18n-tasks.yml

@@ -13,7 +13,7 @@ data:
   ## Provide a custom adapter:
   # adapter: I18n::Tasks::Data::FileSystem
 
-  # Locale files or `Find.find` patterns where translations are read from:
+  # Locale files or `Dir.glob` patterns where translations are read from:
   read:
     ## Default:
     # - config/locales/%{locale}.yml
@@ -92,9 +92,13 @@ search:
   ## - RailsModelMatcher
   ##     Matches ActiveRecord translations like
   ##     User.human_attribute_name(:email) and User.model_name.human
+  ## - DefaultI18nSubjectMatcher
+  ##     Matches ActionMailer's default_i18n_subject method
   ##
   ## To implement your own, please see `I18n::Tasks::Scanners::AstMatchers::BaseMatcher`.
-  # <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
+  # ast_matchers:
+  #   - 'I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher'
+  #   - 'I18n::Tasks::Scanners::AstMatchers::DefaultI18nSubjectMatcher'
 
   ## Multiple scanners can be used. Their results are merged.
   ## The options specified above are passed down to each scanner. Per-scanner options can be specified as well.
@@ -110,9 +114,28 @@ search:
 #   deepl_api_key: "48E92789-57A3-466A-9959-1A1A1A1A1A1A"
 #   # deepl_host: "https://api.deepl.com"
 #   # deepl_version: "v2"
+#   # deepl_glossary_ids:
+#   #   - f28106eb-0e06-489e-82c6-8215d6f95089
+#   #   - 2c6415be-1852-4f54-9e1b-d800463496b4
 #   # add additional options to the DeepL.translate call: https://www.deepl.com/docs-api/translate-text/translate-text/
 #   deepl_options:
 #     formality: prefer_less
+#   # OpenAI
+#   openai_api_key: "sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+#   # openai_model: "gpt-3.5-turbo" # see https://platform.openai.com/docs/models
+#   # may contain `%{from}` and `%{to}`, which will be replaced by source and target locale codes, respectively (using `Kernel.format`)
+#   # openai_system_prompt: >-
+#   #   You are a professional translator that translates content from the %{from} locale
+#   #   to the %{to} locale in an i18n locale array.
+#   #
+#   #   The array has a structured format and contains multiple strings. Your task is to translate
+#   #   each of these strings and create a new array with the translated strings.
+#   #
+#   #   HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+#   #   Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+#   #
+#   #   Keep in mind the context of all the strings for a more accurate translation.
+
 ## Do not consider these keys missing:
 # ignore_missing:
 # - 'errors.messages.{accepted,blank,invalid,too_short,too_long}'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 1.0.13
+  version: 1.0.14
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-14 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,26 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
-- !ruby/object:Gem::Dependency
-  name: better_html
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -388,6 +368,7 @@ files:
 - lib/i18n/tasks/data/file_system.rb
 - lib/i18n/tasks/data/file_system_base.rb
 - lib/i18n/tasks/data/router/conservative_router.rb
+- lib/i18n/tasks/data/router/isolating_router.rb
 - lib/i18n/tasks/data/router/pattern_router.rb
 - lib/i18n/tasks/data/tree/node.rb
 - lib/i18n/tasks/data/tree/nodes.rb
@@ -406,9 +387,9 @@ files:
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
 - lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb
+- lib/i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher.rb
 - lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb
 - lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb
-- lib/i18n/tasks/scanners/erb_ast_processor.rb
 - lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
@@ -474,7 +455,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Manage localization and translation with the awesome power of static analysis

data/lib/i18n/tasks/scanners/erb_ast_processor.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require 'ast'
-require 'set'
-require 'i18n/tasks/scanners/local_ruby_parser'
-
-module I18n::Tasks::Scanners
-  class ErbAstProcessor
-    include AST::Processor::Mixin
-    def initialize
-      super()
-      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
-      @comments = []
-    end
-
-    def process_and_extract_comments(ast)
-      result = process(ast)
-      [result, @comments]
-    end
-
-    def on_code(node)
-      parsed, comments = @ruby_parser.parse(
-        node.children[0],
-        location: node.location
-      )
-      @comments.concat(comments)
-
-      unless parsed.nil?
-        parsed = parsed.updated(
-          nil,
-          parsed.children.map { |child| node?(child) ? process(child) : child }
-        )
-        node = node.updated(:send, parsed)
-      end
-      node
-    end
-
-    # @param node [::Parser::AST::Node]
-    # @return [::Parser::AST::Node]
-    def handler_missing(node)
-      node = handle_comment(node)
-      return if node.nil?
-
-      node.updated(
-        nil,
-        node.children.map { |child| node?(child) ? process(child) : child }
-      )
-    end
-
-    private
-
-    # Convert ERB-comments to ::Parser::Source::Comment and skip processing node
-    #
-    # @param node Parser::AST::Node Potential comment node
-    # @return Parser::AST::Node or nil
-    def handle_comment(node)
-      if node.type == :erb && node.children.size == 4 &&
-         node.children[0]&.type == :indicator && node.children[0].children[0] == '#' &&
-         node.children[2]&.type == :code
-
-        # Do not continue parsing this node
-        comment = node.children[2]
-        @comments << ::Parser::Source::Comment.new(comment.location.expression)
-        return
-      end
-
-      node
-    end
-
-    def node?(node)
-      node.is_a?(::Parser::AST::Node)
-    end
-  end
-end