i18n-tasks 0.9.37 → 1.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +43 -5
  3. data/Rakefile +1 -1
  4. data/config/locales/en.yml +12 -3
  5. data/config/locales/ru.yml +8 -0
  6. data/i18n-tasks.gemspec +14 -7
  7. data/lib/i18n/tasks/cli.rb +8 -7
  8. data/lib/i18n/tasks/command/commander.rb +1 -0
  9. data/lib/i18n/tasks/command/commands/missing.rb +17 -5
  10. data/lib/i18n/tasks/command/options/common.rb +0 -1
  11. data/lib/i18n/tasks/command/options/data.rb +1 -1
  12. data/lib/i18n/tasks/concurrent/cached_value.rb +0 -2
  13. data/lib/i18n/tasks/configuration.rb +13 -7
  14. data/lib/i18n/tasks/data/adapter/yaml_adapter.rb +15 -2
  15. data/lib/i18n/tasks/data/file_formats.rb +1 -1
  16. data/lib/i18n/tasks/data/router/pattern_router.rb +1 -1
  17. data/lib/i18n/tasks/data/tree/node.rb +1 -1
  18. data/lib/i18n/tasks/data/tree/nodes.rb +5 -7
  19. data/lib/i18n/tasks/data/tree/siblings.rb +1 -2
  20. data/lib/i18n/tasks/data/tree/traversal.rb +25 -11
  21. data/lib/i18n/tasks/html_keys.rb +2 -2
  22. data/lib/i18n/tasks/interpolations.rb +7 -3
  23. data/lib/i18n/tasks/key_pattern_matching.rb +2 -0
  24. data/lib/i18n/tasks/locale_pathname.rb +1 -1
  25. data/lib/i18n/tasks/plural_keys.rb +0 -6
  26. data/lib/i18n/tasks/references.rb +3 -3
  27. data/lib/i18n/tasks/reports/base.rb +2 -2
  28. data/lib/i18n/tasks/reports/terminal.rb +3 -3
  29. data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +118 -0
  30. data/lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb +91 -0
  31. data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +69 -0
  32. data/lib/i18n/tasks/scanners/erb_ast_processor.rb +74 -0
  33. data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +48 -0
  34. data/lib/i18n/tasks/scanners/local_ruby_parser.rb +85 -0
  35. data/lib/i18n/tasks/scanners/pattern_mapper.rb +1 -1
  36. data/lib/i18n/tasks/scanners/pattern_scanner.rb +2 -2
  37. data/lib/i18n/tasks/scanners/relative_keys.rb +2 -2
  38. data/lib/i18n/tasks/scanners/results/occurrence.rb +17 -1
  39. data/lib/i18n/tasks/scanners/ruby_ast_call_finder.rb +9 -34
  40. data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb +91 -156
  41. data/lib/i18n/tasks/scanners/ruby_key_literals.rb +2 -2
  42. data/lib/i18n/tasks/split_key.rb +1 -1
  43. data/lib/i18n/tasks/translation.rb +4 -1
  44. data/lib/i18n/tasks/translators/base_translator.rb +17 -12
  45. data/lib/i18n/tasks/translators/deepl_translator.rb +34 -11
  46. data/lib/i18n/tasks/translators/google_translator.rb +1 -1
  47. data/lib/i18n/tasks/translators/openai_translator.rb +100 -0
  48. data/lib/i18n/tasks/used_keys.rb +9 -6
  49. data/lib/i18n/tasks/version.rb +1 -1
  50. data/lib/i18n/tasks.rb +11 -0
  51. data/templates/config/i18n-tasks.yml +17 -4
  52. data/templates/minitest/i18n_test.rb +6 -6
  53. metadata +74 -16
@@ -1,56 +1,31 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'ast'
4
- require 'set'
4
+
5
5
  module I18n::Tasks::Scanners
6
6
  class RubyAstCallFinder
7
7
  include AST::Processor::Mixin
8
8
 
9
- # @param receiver_messages [Set<Pair<[nil, AST::Node>, Symbol>>] The receiver-message pairs to look for.
10
- def initialize(receiver_messages:)
11
- super()
12
- @message_receivers = receiver_messages.each_with_object({}) do |(receiver, message), t|
13
- (t[message] ||= []) << receiver
14
- end
15
- end
16
-
17
9
  # @param root_node [Parser::AST:Node]
18
- # @yieldparam send_node [Parser::AST:Node]
19
- # @yieldparam method_name [nil, String] the surrounding method's name.
20
- def find_calls(root_node, &block)
21
- @callback = block
22
- process root_node
23
- ensure
24
- @callback = nil
25
- end
26
-
27
- # @param root_node (see #find_calls)
28
- # @yieldparam (see #find_calls)
29
- # @return [Array<block return values excluding nils>]
10
+ # @return [Pair<Parser::AST::Node, method_name as string>] for all nodes with :send type
30
11
  def collect_calls(root_node)
31
- results = []
32
- find_calls root_node do |send_node, method_name|
33
- result = yield send_node, method_name
34
- results << result if result
35
- end
36
- results
12
+ @results = []
13
+ process(root_node)
14
+ @results
37
15
  end
38
16
 
39
17
  def on_def(node)
40
18
  @method_name = node.children[0]
41
- handler_missing node
19
+ handler_missing(node)
42
20
  ensure
43
21
  @method_name = nil
44
22
  end
45
23
 
46
24
  def on_send(send_node)
47
- receiver = send_node.children[0]
48
- message = send_node.children[1]
49
- valid_receivers = @message_receivers[message]
50
- # use `any?` because `include?` checks type equality, but the receiver is a Parser::AST::Node != AST::Node.
51
- @callback.call(send_node, @method_name) if valid_receivers&.any? { |r| r == receiver }
25
+ @results << [send_node, @method_name]
26
+
52
27
  # always invoke handler_missing to get nested translations in children
53
- handler_missing send_node
28
+ handler_missing(send_node)
54
29
  nil
55
30
  end
56
31
 
@@ -3,27 +3,23 @@
3
3
  require 'i18n/tasks/scanners/file_scanner'
4
4
  require 'i18n/tasks/scanners/relative_keys'
5
5
  require 'i18n/tasks/scanners/ruby_ast_call_finder'
6
+ require 'i18n/tasks/scanners/ast_matchers/message_receivers_matcher'
7
+ require 'i18n/tasks/scanners/ast_matchers/rails_model_matcher'
6
8
  require 'parser/current'
7
9
 
8
- # rubocop:disable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity
9
- # TODO: make this class more readable.
10
-
11
10
  module I18n::Tasks::Scanners
12
11
  # Scan for I18n.translate calls using whitequark/parser
13
- class RubyAstScanner < FileScanner # rubocop:disable Metrics/ClassLength
12
+ class RubyAstScanner < FileScanner
14
13
  include RelativeKeys
15
14
  include AST::Sexp
16
15
 
17
16
  MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/.freeze
18
- RECEIVER_MESSAGES = [nil, AST::Node.new(:const, [nil, :I18n])].product(%i[t t! translate translate!])
19
17
 
20
18
  def initialize(**args)
21
19
  super(**args)
22
20
  @parser = ::Parser::CurrentRuby.new
23
21
  @magic_comment_parser = ::Parser::CurrentRuby.new
24
- @call_finder = RubyAstCallFinder.new(
25
- receiver_messages: config[:receiver_messages] || RECEIVER_MESSAGES
26
- )
22
+ @matchers = setup_matchers
27
23
  end
28
24
 
29
25
  protected
@@ -32,178 +28,117 @@ module I18n::Tasks::Scanners
32
28
  #
33
29
  # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
34
30
  def scan_file(path)
35
- @parser.reset
36
- ast, comments = @parser.parse_with_comments(make_buffer(path))
37
-
38
- results = @call_finder.collect_calls ast do |send_node, method_name|
39
- send_node_to_key_occurrence(send_node, method_name)
40
- end
31
+ ast, comments = path_to_ast_and_comments(path)
41
32
 
42
- magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
43
- comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
44
- # transform_values is only available in ActiveSupport 4.2+
45
- h.each { |k, v| h[k] = v.first }
46
- end.invert
47
- results + (magic_comments.flat_map do |comment|
48
- @parser.reset
49
- associated_node = comment_to_node[comment]
50
- @call_finder.collect_calls(
51
- @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
52
- ) do |send_node, _method_name|
53
- # method_name is not available at this stage
54
- send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
55
- end
56
- end)
33
+ ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
57
34
  rescue Exception => e # rubocop:disable Lint/RescueException
58
35
  raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
59
36
  end
60
37
 
61
- # @param send_node [Parser::AST::Node]
62
- # @param method_name [Symbol, nil]
63
- # @param location [Parser::Source::Map]
64
- # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
65
- def send_node_to_key_occurrence(send_node, method_name, location: send_node.loc)
66
- if (first_arg_node = send_node.children[2]) &&
67
- (key = extract_string(first_arg_node))
68
- if (second_arg_node = send_node.children[3]) &&
69
- second_arg_node.type == :hash
70
- if (scope_node = extract_hash_pair(second_arg_node, 'scope'))
71
- scope = extract_string(scope_node.children[1],
72
- array_join_with: '.', array_flatten: true, array_reject_blank: true)
73
- return nil if scope.nil? && scope_node.type != :nil
74
-
75
- key = [scope, key].join('.') unless scope == ''
76
- end
77
- default_arg = if (default_arg_node = extract_hash_pair(second_arg_node, 'default'))
78
- extract_string(default_arg_node.children[1])
79
- end
80
- end
81
- full_key = if send_node.children[0].nil?
82
- # Relative keys only work if called via `t()` but not `I18n.t()`:
83
- absolute_key(key, location.expression.source_buffer.name, calling_method: method_name)
84
- else
85
- key
86
- end
87
- [full_key, range_to_occurrence(key, location.expression, default_arg: default_arg)]
88
- end
38
+ # Parse file on path and returns AST and comments.
39
+ #
40
+ # @param path Path to file to parse
41
+ # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
42
+ def path_to_ast_and_comments(path)
43
+ @parser.reset
44
+ @parser.parse_with_comments(make_buffer(path))
89
45
  end
90
46
 
91
- # Extract a hash pair with a given literal key.
92
- #
93
- # @param node [AST::Node] a node of type `:hash`.
94
- # @param key [String] node key as a string (indifferent symbol-string matching).
95
- # @return [AST::Node, nil] a node of type `:pair` or nil.
96
- def extract_hash_pair(node, key)
97
- node.children.detect do |child|
98
- next unless child.type == :pair
99
-
100
- key_node = child.children[0]
101
- %i[sym str].include?(key_node.type) && key_node.children[0].to_s == key
102
- end
47
+ def keys_relative_to_calling_method?(path)
48
+ /controllers|mailers/.match(path)
103
49
  end
104
50
 
105
- # If the node type is of `%i(sym str int false true)`, return the value as a string.
106
- # Otherwise, if `config[:strict]` is `false` and the type is of `%i(dstr dsym)`,
107
- # return the source as if it were a string.
51
+ # Create an {Parser::Source::Buffer} with the given contents.
52
+ # The contents are assigned a {Parser::Source::Buffer#raw_source}.
108
53
  #
109
- # @param node [Parser::AST::Node]
110
- # @param array_join_with [String, nil] if set to a string, arrays will be processed and their elements joined.
111
- # @param array_flatten [Boolean] if true, nested arrays are flattened,
112
- # otherwise their source is copied and surrounded by #{}. No effect unless `array_join_with` is set.
113
- # @param array_reject_blank [Boolean] if true, empty strings and `nil`s are skipped.
114
- # No effect unless `array_join_with` is set.
115
- # @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode
116
- # or the node type is not supported.
117
- def extract_string(node, array_join_with: nil, array_flatten: false, array_reject_blank: false)
118
- if %i[sym str int].include?(node.type)
119
- node.children[0].to_s
120
- elsif %i[true false].include?(node.type)
121
- node.type.to_s
122
- elsif node.type == :nil
123
- ''
124
- elsif node.type == :array && array_join_with
125
- extract_array_as_string(
126
- node,
127
- array_join_with: array_join_with,
128
- array_flatten: array_flatten,
129
- array_reject_blank: array_reject_blank
130
- ).tap do |str|
131
- # `nil` is returned when a dynamic value is encountered in strict mode. Propagate:
132
- return nil if str.nil?
133
- end
134
- elsif !config[:strict] && %i[dsym dstr].include?(node.type)
135
- node.children.map do |child|
136
- if %i[sym str].include?(child.type)
137
- child.children[0].to_s
138
- else
139
- child.loc.expression.source
140
- end
141
- end.join
54
+ # @param path [String] Path to assign as the buffer name.
55
+ # @param contents [String]
56
+ # @return [Parser::Source::Buffer] file contents
57
+ def make_buffer(path, contents = read_file(path))
58
+ Parser::Source::Buffer.new(path).tap do |buffer|
59
+ buffer.raw_source = contents
142
60
  end
143
61
  end
144
62
 
145
- # Extract an array as a single string.
63
+ # Convert an array of {Parser::Source::Comment} to occurrences.
146
64
  #
147
- # @param array_join_with [String] joiner of the array elements.
148
- # @param array_flatten [Boolean] if true, nested arrays are flattened,
149
- # otherwise their source is copied and surrounded by #{}.
150
- # @param array_reject_blank [Boolean] if true, empty strings and `nil`s are skipped.
151
- # @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode.
152
- def extract_array_as_string(node, array_join_with:, array_flatten: false, array_reject_blank: false)
153
- children_strings = node.children.map do |child|
154
- if %i[sym str int true false].include?(child.type)
155
- extract_string child
156
- else
157
- # ignore dynamic argument in strict mode
158
- return nil if config[:strict]
159
-
160
- if %i[dsym dstr].include?(child.type) || (child.type == :array && array_flatten)
161
- extract_string(child, array_join_with: array_join_with)
162
- else
163
- "\#{#{child.loc.expression.source}}"
65
+ # @param path Path to file
66
+ # @param ast Parser::AST::Node
67
+ # @param comments [Parser::Source::Comment]
68
+ # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
69
+ def comments_to_occurences(path, ast, comments)
70
+ magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
71
+ comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
72
+ h.transform_values!(&:first)
73
+ end.invert
74
+
75
+ magic_comments.flat_map do |comment|
76
+ @parser.reset
77
+ associated_node = comment_to_node[comment]
78
+ ast = @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
79
+ calls = RubyAstCallFinder.new.collect_calls(ast)
80
+ results = []
81
+
82
+ # method_name is not available at this stage
83
+ calls.each do |send_node, _method_name|
84
+ @matchers.each do |matcher|
85
+ result = matcher.convert_to_key_occurrences(
86
+ send_node,
87
+ nil,
88
+ location: associated_node || comment.location
89
+ )
90
+ results << result if result
164
91
  end
165
92
  end
93
+
94
+ results
166
95
  end
167
- if array_reject_blank
168
- children_strings.reject! do |x|
169
- # empty strings and nils in the scope argument are ignored by i18n
170
- x == ''
96
+ end
97
+
98
+ # Convert {Parser::AST::Node} to occurrences.
99
+ #
100
+ # @param ast {Parser::Source::Comment}
101
+ # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
102
+ def ast_to_occurences(ast)
103
+ calls = RubyAstCallFinder.new.collect_calls(ast)
104
+ results = []
105
+ calls.each do |send_node, method_name|
106
+ @matchers.each do |matcher|
107
+ result = matcher.convert_to_key_occurrences(send_node, method_name)
108
+ results << result if result
171
109
  end
172
110
  end
173
- children_strings.join(array_join_with)
174
- end
175
111
 
176
- def keys_relative_to_calling_method?(path)
177
- /controllers|mailers/.match(path)
112
+ results
178
113
  end
179
114
 
180
- # @param raw_key [String]
181
- # @param range [Parser::Source::Range]
182
- # @param default_arg [String, nil]
183
- # @return [Results::Occurrence]
184
- def range_to_occurrence(raw_key, range, default_arg: nil)
185
- Results::Occurrence.new(
186
- path: range.source_buffer.name,
187
- pos: range.begin_pos,
188
- line_num: range.line,
189
- line_pos: range.column,
190
- line: range.source_line,
191
- raw_key: raw_key,
192
- default_arg: default_arg
193
- )
194
- end
115
+ def setup_matchers
116
+ if config[:receiver_messages]
117
+ config[:receiver_messages].map do |receiver, message|
118
+ AstMatchers::MessageReceiversMatcher.new(
119
+ receivers: [receiver],
120
+ message: message,
121
+ scanner: self
122
+ )
123
+ end
124
+ else
125
+ matchers = %i[t t! translate translate!].map do |message|
126
+ AstMatchers::MessageReceiversMatcher.new(
127
+ receivers: [
128
+ AST::Node.new(:const, [nil, :I18n]),
129
+ nil
130
+ ],
131
+ message: message,
132
+ scanner: self
133
+ )
134
+ end
195
135
 
196
- # Create an {Parser::Source::Buffer} with the given contents.
197
- # The contents are assigned a {Parser::Source::Buffer#raw_source}.
198
- #
199
- # @param path [String] Path to assign as the buffer name.
200
- # @param contents [String]
201
- # @return [Parser::Source::Buffer] file contents
202
- def make_buffer(path, contents = read_file(path))
203
- Parser::Source::Buffer.new(path).tap do |buffer|
204
- buffer.raw_source = contents
136
+ Array(config[:ast_matchers]).each do |class_name|
137
+ matchers << ActiveSupport::Inflector.constantize(class_name).new(scanner: self)
138
+ end
139
+
140
+ matchers
205
141
  end
206
142
  end
207
143
  end
208
144
  end
209
- # rubocop:enable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity
@@ -15,12 +15,12 @@ module I18n::Tasks::Scanners
15
15
  # @param literal [String] e.g: "key", 'key', or :key.
16
16
  # @return [String] key
17
17
  def strip_literal(literal)
18
- literal = literal[1..-1] if literal[0] == ':'
18
+ literal = literal[1..] if literal[0] == ':'
19
19
  literal = literal[1..-2] if literal[0] == "'" || literal[0] == '"'
20
20
  literal
21
21
  end
22
22
 
23
- VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!:;À-ž\/])/.freeze
23
+ VALID_KEY_CHARS = %r{(?:[[:word:]]|[-.?!:;À-ž/])}.freeze
24
24
  VALID_KEY_RE = /^#{VALID_KEY_CHARS}+$/.freeze
25
25
 
26
26
  def valid_key?(key)
@@ -20,7 +20,7 @@ module I18n
20
20
  parts << part
21
21
  pos += part.length + 1
22
22
  if parts.length + 1 >= max
23
- parts << key[pos..-1] unless pos == key.length
23
+ parts << key[pos..] unless pos == key.length
24
24
  break
25
25
  end
26
26
  end
@@ -2,13 +2,14 @@
2
2
 
3
3
  require 'i18n/tasks/translators/deepl_translator'
4
4
  require 'i18n/tasks/translators/google_translator'
5
+ require 'i18n/tasks/translators/openai_translator'
5
6
  require 'i18n/tasks/translators/yandex_translator'
6
7
 
7
8
  module I18n::Tasks
8
9
  module Translation
9
10
  # @param [I18n::Tasks::Tree::Siblings] forest to translate to the locales of its root nodes
10
11
  # @param [String] from locale
11
- # @param [:deepl, :google, :yandex] backend
12
+ # @param [:deepl, :openai, :google, :yandex] backend
12
13
  # @return [I18n::Tasks::Tree::Siblings] translated forest
13
14
  def translate_forest(forest, from:, backend: :google)
14
15
  case backend
@@ -16,6 +17,8 @@ module I18n::Tasks
16
17
  Translators::DeeplTranslator.new(self).translate_forest(forest, from)
17
18
  when :google
18
19
  Translators::GoogleTranslator.new(self).translate_forest(forest, from)
20
+ when :openai
21
+ Translators::OpenAiTranslator.new(self).translate_forest(forest, from)
19
22
  when :yandex
20
23
  Translators::YandexTranslator.new(self).translate_forest(forest, from)
21
24
  else
@@ -3,6 +3,7 @@
3
3
  module I18n::Tasks
4
4
  module Translators
5
5
  class BaseTranslator
6
+ include ::I18n::Tasks::Logging
6
7
  # @param [I18n::Tasks::BaseTask] i18n_tasks
7
8
  def initialize(i18n_tasks)
8
9
  @i18n_tasks = i18n_tasks
@@ -31,7 +32,7 @@ module I18n::Tasks
31
32
  reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
32
33
  list -= reference_key_vals
33
34
  result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
34
- fetch_translations list_slice, opts.merge(is_html ? options_for_html : options_for_plain)
35
+ fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
35
36
  end.reduce(:+) || []
36
37
  result.concat(reference_key_vals)
37
38
  result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
@@ -41,34 +42,36 @@ module I18n::Tasks
41
42
  # @param [Array<[String, Object]>] list of key-value pairs
42
43
  # @return [Array<[String, Object]>] translated list
43
44
  def fetch_translations(list, opts)
44
- from_values(list, translate_values(to_values(list), **options_for_translate_values(**opts))).tap do |result|
45
+ options = options_for_translate_values(**opts)
46
+ from_values(list, translate_values(to_values(list, options), **options), options).tap do |result|
45
47
  fail CommandError, no_results_error_message if result.blank?
46
48
  end
47
49
  end
48
50
 
49
51
  # @param [Array<[String, Object]>] list of key-value pairs
50
52
  # @return [Array<String>] values for translation extracted from list
51
- def to_values(list)
52
- list.map { |l| dump_value l[1] }.flatten.compact
53
+ def to_values(list, opts)
54
+ list.map { |l| dump_value(l[1], opts) }.flatten.compact
53
55
  end
54
56
 
55
57
  # @param [Array<[String, Object]>] list
56
58
  # @param [Array<String>] translated_values
57
59
  # @return [Array<[String, Object]>] translated key-value pairs
58
- def from_values(list, translated_values)
60
+ def from_values(list, translated_values, opts)
59
61
  keys = list.map(&:first)
60
62
  untranslated_values = list.map(&:last)
61
- keys.zip parse_value(untranslated_values, translated_values.to_enum)
63
+ keys.zip parse_value(untranslated_values, translated_values.to_enum, opts)
62
64
  end
63
65
 
64
66
  # Prepare value for translation.
65
67
  # @return [String, Array<String, nil>, nil] value for Google Translate or nil for non-string values
66
- def dump_value(value)
68
+ def dump_value(value, opts)
67
69
  case value
68
70
  when Array
69
71
  # dump recursively
70
- value.map { |v| dump_value v }
72
+ value.map { |v| dump_value(v, opts) }
71
73
  when String
74
+ value = CGI.escapeHTML(value) if opts[:html_escape]
72
75
  replace_interpolations value unless value.empty?
73
76
  end
74
77
  end
@@ -77,16 +80,18 @@ module I18n::Tasks
77
80
  # @param [Object] untranslated
78
81
  # @param [Enumerator] each_translated
79
82
  # @return [Object] final translated value
80
- def parse_value(untranslated, each_translated)
83
+ def parse_value(untranslated, each_translated, opts)
81
84
  case untranslated
82
85
  when Array
83
86
  # implode array
84
- untranslated.map { |from| parse_value(from, each_translated) }
87
+ untranslated.map { |from| parse_value(from, each_translated, opts) }
85
88
  when String
86
89
  if untranslated.empty?
87
90
  untranslated
88
91
  else
89
- restore_interpolations untranslated, each_translated.next
92
+ value = each_translated.next
93
+ value = CGI.unescapeHTML(value) if opts[:html_escape]
94
+ restore_interpolations(untranslated, value)
90
95
  end
91
96
  else
92
97
  untranslated
@@ -114,7 +119,7 @@ module I18n::Tasks
114
119
 
115
120
  values = untranslated.scan(INTERPOLATION_KEY_RE)
116
121
  translated.gsub(/#{Regexp.escape(UNTRANSLATABLE_STRING)}\d+/i) do |m|
117
- values[m[UNTRANSLATABLE_STRING.length..-1].to_i]
122
+ values[m[UNTRANSLATABLE_STRING.length..].to_i]
118
123
  end
119
124
  rescue StandardError => e
120
125
  raise_interpolation_error(untranslated, translated, e)
@@ -4,6 +4,11 @@ require 'i18n/tasks/translators/base_translator'
4
4
 
5
5
  module I18n::Tasks::Translators
6
6
  class DeeplTranslator < BaseTranslator
7
+ # max allowed texts per request
8
+ BATCH_SIZE = 50
9
+ # those languages must be specified with their sub-kind e.g en-us
10
+ SPECIFIC_TARGETS = %w[en pt].freeze
11
+
7
12
  def initialize(*)
8
13
  begin
9
14
  require 'deepl'
@@ -17,16 +22,22 @@ module I18n::Tasks::Translators
17
22
  protected
18
23
 
19
24
  def translate_values(list, from:, to:, **options)
20
- result = DeepL.translate(list, to_deepl_compatible_locale(from), to_deepl_compatible_locale(to), options)
21
- if result.is_a?(DeepL::Resources::Text)
22
- [result.text]
23
- else
24
- result.map(&:text)
25
+ results = []
26
+ list.each_slice(BATCH_SIZE) do |parts|
27
+ res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options)
28
+ if res.is_a?(DeepL::Resources::Text)
29
+ results << res.text
30
+ else
31
+ results += res.map(&:text)
32
+ end
25
33
  end
34
+ results
26
35
  end
27
36
 
28
37
  def options_for_translate_values(**options)
29
- { ignore_tags: %w[i18n] }.merge(options)
38
+ extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
39
+
40
+ extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
30
41
  end
31
42
 
32
43
  def options_for_html
@@ -34,7 +45,7 @@ module I18n::Tasks::Translators
34
45
  end
35
46
 
36
47
  def options_for_plain
37
- { preserve_formatting: true }
48
+ { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
38
49
  end
39
50
 
40
51
  # @param [String] value
@@ -60,22 +71,34 @@ module I18n::Tasks::Translators
60
71
 
61
72
  private
62
73
 
63
- # Convert 'es-ES' to 'ES'
64
- def to_deepl_compatible_locale(locale)
74
+ # Convert 'es-ES' to 'ES', en-us to EN
75
+ def to_deepl_source_locale(locale)
65
76
  locale.to_s.split('-', 2).first.upcase
66
77
  end
67
78
 
79
+ # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
80
+ def to_deepl_target_locale(locale)
81
+ loc, sub = locale.to_s.split('-')
82
+ if SPECIFIC_TARGETS.include?(loc)
83
+ # Must see how the deepl api evolves, so this could be an error in the future
84
+ warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
85
+ locale.to_s.upcase
86
+ else
87
+ loc.upcase
88
+ end
89
+ end
90
+
68
91
  def configure_api_key!
69
92
  api_key = @i18n_tasks.translation_config[:deepl_api_key]
70
93
  host = @i18n_tasks.translation_config[:deepl_host]
71
94
  version = @i18n_tasks.translation_config[:deepl_version]
72
95
  fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.no_api_key') if api_key.blank?
73
96
 
74
- DeepL.configure { |config|
97
+ DeepL.configure do |config|
75
98
  config.auth_key = api_key
76
99
  config.host = host unless host.blank?
77
100
  config.version = version unless version.blank?
78
- }
101
+ end
79
102
  end
80
103
  end
81
104
  end
@@ -55,7 +55,7 @@ module I18n::Tasks::Translators
55
55
  key = @i18n_tasks.translation_config[:google_translate_api_key]
56
56
  # fallback with deprecation warning
57
57
  if @i18n_tasks.translation_config[:api_key]
58
- @i18n_tasks.warn_deprecated(
58
+ warn_deprecated(
59
59
  'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
60
60
  )
61
61
  key ||= translation_config[:api_key]