i18n-tasks 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +321 -242
  3. data/Rakefile +5 -1
  4. data/benchmarks/bench_helper.rb +156 -0
  5. data/benchmarks/data_bench.rb +173 -0
  6. data/benchmarks/end_to_end_bench.rb +138 -0
  7. data/benchmarks/fixtures/generator.rb +257 -0
  8. data/benchmarks/results/baseline.json +250 -0
  9. data/benchmarks/run_all.rb +60 -0
  10. data/benchmarks/scanning_bench.rb +135 -0
  11. data/benchmarks/tree_bench.rb +205 -0
  12. data/config/locales/en.yml +10 -0
  13. data/config/locales/ru.yml +14 -0
  14. data/i18n-tasks.gemspec +1 -1
  15. data/lib/i18n/tasks/base_task.rb +2 -0
  16. data/lib/i18n/tasks/command/commands/check_prism.rb +67 -15
  17. data/lib/i18n/tasks/command/commands/prune.rb +57 -0
  18. data/lib/i18n/tasks/commands.rb +2 -0
  19. data/lib/i18n/tasks/data/adapter/json_adapter.rb +1 -1
  20. data/lib/i18n/tasks/data/file_formats.rb +4 -1
  21. data/lib/i18n/tasks/data/router/conservative_router.rb +13 -22
  22. data/lib/i18n/tasks/data/router/pattern_router.rb +26 -14
  23. data/lib/i18n/tasks/data/tree/node.rb +3 -3
  24. data/lib/i18n/tasks/data/tree/siblings.rb +11 -3
  25. data/lib/i18n/tasks/data/tree/traversal.rb +20 -14
  26. data/lib/i18n/tasks/interpolations.rb +61 -3
  27. data/lib/i18n/tasks/logging.rb +1 -2
  28. data/lib/i18n/tasks/missing_keys.rb +7 -38
  29. data/lib/i18n/tasks/plural_keys.rb +56 -0
  30. data/lib/i18n/tasks/prune.rb +18 -0
  31. data/lib/i18n/tasks/references.rb +9 -1
  32. data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +11 -2
  33. data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +16 -2
  34. data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +7 -3
  35. data/lib/i18n/tasks/scanners/file_scanner.rb +4 -1
  36. data/lib/i18n/tasks/scanners/files/file_finder.rb +29 -6
  37. data/lib/i18n/tasks/scanners/files/parse_cache.rb +24 -0
  38. data/lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb +33 -0
  39. data/lib/i18n/tasks/scanners/prism_scanners/nodes.rb +57 -23
  40. data/lib/i18n/tasks/scanners/prism_scanners/visitor.rb +70 -16
  41. data/lib/i18n/tasks/scanners/relative_keys.rb +21 -11
  42. data/lib/i18n/tasks/scanners/ruby_scanner.rb +38 -5
  43. data/lib/i18n/tasks/split_key.rb +16 -6
  44. data/lib/i18n/tasks/translators/base_translator.rb +25 -23
  45. data/lib/i18n/tasks/translators/deepl_translator.rb +18 -15
  46. data/lib/i18n/tasks/translators/google_translator.rb +41 -14
  47. data/lib/i18n/tasks/translators/openai_translator.rb +31 -17
  48. data/lib/i18n/tasks/translators/watsonx_translator.rb +8 -10
  49. data/lib/i18n/tasks/translators/yandex_translator.rb +4 -0
  50. data/lib/i18n/tasks/unused_keys.rb +35 -5
  51. data/lib/i18n/tasks/used_keys.rb +6 -1
  52. data/lib/i18n/tasks/version.rb +1 -1
  53. data/script/report_test_results.rb +58 -0
  54. data/templates/config/i18n-tasks.yml +11 -3
  55. metadata +17 -7
@@ -4,7 +4,7 @@
4
4
  # Used in the PrismScanners::Visitor class.
5
5
  module I18n::Tasks::Scanners::PrismScanners
6
6
  class Root
7
- attr_reader(:calls, :translation_calls, :children, :node, :parent, :rails, :file_path)
7
+ attr_reader(:calls, :translation_calls, :children, :node, :parent, :rails)
8
8
 
9
9
  def initialize(node: nil, parent: nil, file_path: nil, rails: false)
10
10
  @calls = []
@@ -16,6 +16,10 @@ module I18n::Tasks::Scanners::PrismScanners
16
16
  @file_path = file_path
17
17
  end
18
18
 
19
+ def file_path
20
+ @file_path || @parent&.file_path
21
+ end
22
+
19
23
  def add_child(node)
20
24
  @children << node
21
25
  node
@@ -30,15 +34,19 @@ module I18n::Tasks::Scanners::PrismScanners
30
34
  end
31
35
 
32
36
  def rails_view?
33
- rails && file_path.present? && file_path.include?("app/views/")
34
- end
37
+ return false unless rails && file_path.present?
35
38
 
36
- def partial_view?
37
- file_path.present? && File.basename(file_path).start_with?("_")
39
+ if file_path.include?("app/views/")
40
+ true
41
+ elsif file_path.include?("app/components/")
42
+ !file_path.end_with?(".rb")
43
+ else
44
+ false
45
+ end
38
46
  end
39
47
 
40
48
  def support_relative_keys?
41
- rails_view? && !partial_view?
49
+ rails_view?
42
50
  end
43
51
 
44
52
  def support_candidate_keys?
@@ -47,8 +55,15 @@ module I18n::Tasks::Scanners::PrismScanners
47
55
 
48
56
  def path
49
57
  if rails_view?
50
- folder_path = file_path.sub(%r{app/views/}, "").split("/")
58
+ folder_path = if file_path.include?("app/views/")
59
+ file_path.sub(%r{app/views/}, "").split("/")
60
+ else
61
+ file_path.sub(%r{app/components/}, "").split("/")
62
+ end
63
+
51
64
  name = folder_path.pop.split(".").first
65
+ # Remove leading underscores from partials
66
+ name = name[1..] if name.start_with?("_")
52
67
 
53
68
  [*folder_path, name]
54
69
  else
@@ -121,7 +136,7 @@ module I18n::Tasks::Scanners::PrismScanners
121
136
  # Example: parent.path = ["events", "create"], key = ".success"
122
137
  # yields: ["events.create.success", "events.success"]
123
138
  parent_path = parent&.path || []
124
- rel_key = key[1..] # strip leading dot # rubocop:disable Performance/ArraySemiInfiniteRangeSlice
139
+ rel_key = key[1..] # strip leading dot
125
140
 
126
141
  candidates = []
127
142
  parent_path_length = parent_path.length
@@ -134,15 +149,15 @@ module I18n::Tasks::Scanners::PrismScanners
134
149
  candidates.map { |c| c.gsub("..", ".") }
135
150
  elsif relative_key?
136
151
  # For relative keys in views, just append to the full path
137
- [base_parts + parent.path + [key[1..]]].flatten.compact.join(".").gsub("..", ".") # rubocop:disable Performance/ChainArrayAllocation
152
+ [base_parts + parent.path + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
138
153
  elsif key.start_with?(".")
139
- [base_parts + [key[1..]]].flatten.compact.join(".").gsub("..", ".") # rubocop:disable Performance/ArraySemiInfiniteRangeSlice,Performance/ChainArrayAllocation
154
+ [base_parts + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
140
155
  elsif @candidate_keys.present?
141
156
  ([key] + @candidate_keys).map do |c|
142
- [base_parts + [c]].flatten.compact.join(".").gsub("..", ".") # rubocop:disable Performance/ChainArrayAllocation
157
+ [base_parts + [c]].flatten.compact.join(".").gsub("..", ".")
143
158
  end
144
159
  else
145
- [base_parts + [key]].flatten.compact.join(".").gsub("..", ".") # rubocop:disable Performance/ChainArrayAllocation
160
+ [base_parts + [key]].flatten.compact.join(".").gsub("..", ".")
146
161
  end
147
162
  end
148
163
 
@@ -150,26 +165,37 @@ module I18n::Tasks::Scanners::PrismScanners
150
165
 
151
166
  def scope
152
167
  return nil if @options.nil?
153
- return nil unless @options["scope"]
168
+ return nil unless @options.key?("scope")
169
+
170
+ scope_value = @options["scope"]
171
+ scope_array = Array(scope_value)
154
172
 
155
- fail(ScopeError, "Could not process scope") if @options.key?("scope") && (Array(@options["scope"]).empty? || !Array(@options["scope"]).all? { |s| s.is_a?(String) || s.is_a?(Symbol) })
173
+ # ArgumentsVisitor returns a Prism::LocalVariableReadNode for local variables.
174
+ # Scope cannot be resolved if array is empty
175
+ if scope_value.is_a?(Prism::LocalVariableReadNode) ||
176
+ scope_array.empty? ||
177
+ !scope_array.all? { |s| s.is_a?(String) || s.is_a?(Symbol) }
178
+ fail(ScopeError, "Could not process scope")
179
+ end
156
180
 
157
- Array(@options["scope"]).join(".")
181
+ scope_array.join(".")
158
182
  end
159
183
 
160
184
  def occurrence(file_path)
161
- local_node = @node
162
-
163
- location = local_node.location
185
+ location = @node.location
164
186
 
165
187
  final = full_key
166
188
  return nil if final.nil?
167
189
 
190
+ # A zero-length copy slices only the first line, not every line the node spans.
191
+ line = location.copy(length: 0).slice_lines.chomp
192
+
168
193
  occurrence = ::I18n::Tasks::Scanners::Results::Occurrence.new(
169
194
  path: file_path,
170
- line: local_node.respond_to?(:slice) ? local_node.slice : local_node.location.slice,
171
- pos: location.start_offset,
172
- line_pos: location.start_column,
195
+ line: line,
196
+ pos: location.start_character_offset,
197
+ # start_character_column counts the characters from the start of the file, twice.
198
+ line_pos: line.byteslice(0, location.start_column).length,
173
199
  line_num: location.start_line,
174
200
  raw_key: key,
175
201
  candidate_keys: Array(final)
@@ -304,7 +330,7 @@ module I18n::Tasks::Scanners::PrismScanners
304
330
  end
305
331
 
306
332
  def support_relative_keys?
307
- controller? || mailer?
333
+ controller? || mailer? || view_component?
308
334
  end
309
335
 
310
336
  def support_candidate_keys?
@@ -323,6 +349,10 @@ module I18n::Tasks::Scanners::PrismScanners
323
349
  @rails && @node.name.to_s.end_with?("Mailer")
324
350
  end
325
351
 
352
+ def view_component?
353
+ @rails && file_path.present? && file_path.include?("app/components/")
354
+ end
355
+
326
356
  def path_name
327
357
  path = @node.constant_path.full_name_parts.map { |s| s.to_s.underscore }
328
358
  path.last.delete_suffix!("_controller") if controller?
@@ -345,7 +375,11 @@ module I18n::Tasks::Scanners::PrismScanners
345
375
  delegate(:support_candidate_keys?, to: :parent)
346
376
 
347
377
  def path
348
- (@parent&.path || []) + [@node.name]
378
+ if @parent.respond_to?(:view_component?) && @parent.view_component?
379
+ @parent.path
380
+ else
381
+ (@parent&.path || []) + [@node.name]
382
+ end
349
383
  end
350
384
 
351
385
  def name
@@ -14,7 +14,7 @@ module I18n::Tasks::Scanners::PrismScanners
14
14
  class Visitor < Prism::Visitor # rubocop:disable Metrics/ClassLength
15
15
  MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/
16
16
 
17
- attr_reader(:calls, :current_module, :current_class, :current_method, :root)
17
+ attr_reader(:calls, :current_module, :current_class, :current_method, :root, :processed_magic_comment_ids)
18
18
 
19
19
  def initialize(rails: false, file_path: nil)
20
20
  @calls = []
@@ -23,6 +23,7 @@ module I18n::Tasks::Scanners::PrismScanners
23
23
  @current_class = nil
24
24
  @current_method = nil
25
25
  @root = Root.new(file_path:, rails: rails)
26
+ @processed_magic_comment_ids = []
26
27
 
27
28
  @rails = rails
28
29
 
@@ -118,6 +119,26 @@ module I18n::Tasks::Scanners::PrismScanners
118
119
  super
119
120
  end
120
121
 
122
+ def visit_keyword_hash_node(node)
123
+ handle_comments(node)
124
+ super
125
+ end
126
+
127
+ def visit_statements_node(node)
128
+ handle_comments(node)
129
+ super
130
+ end
131
+
132
+ def visit_local_variable_write_node(node)
133
+ handle_comments(node)
134
+ super
135
+ end
136
+
137
+ def visit_symbol_node(node)
138
+ handle_comments(node)
139
+ super
140
+ end
141
+
121
142
  def process
122
143
  @root.process
123
144
  end
@@ -147,27 +168,38 @@ module I18n::Tasks::Scanners::PrismScanners
147
168
 
148
169
  next if match.nil?
149
170
 
171
+ @processed_magic_comment_ids << comment.object_id
172
+
150
173
  string =
151
174
  content.gsub(MAGIC_COMMENT_PREFIX, "").delete("#").strip
175
+ parse_result = Prism.parse(string)
176
+ next if parse_result.respond_to?(:errors) && parse_result.errors.any?
177
+
152
178
  visitor = Visitor.new
153
- Prism
154
- .parse(string)
155
- .value
156
- .accept(visitor)
179
+ parse_result.value.accept(visitor)
157
180
 
158
181
  # Process and remap the found translation calls to be for the found comment
159
182
  visitor.process.each do |comment_node|
160
- parent.add_translation_call(comment_node.with_node(node))
183
+ parent.add_translation_call(comment_node.with_parent(parent).with_node(commented_node(node, comment)))
161
184
  end
162
185
  end
163
186
  end
164
187
 
188
+ # Prism 2.0 attaches a comment on its own line to the enclosing statements,
189
+ # Prism 1.x attached it to the next statement, or the previous one at the end.
190
+ def commented_node(node, comment)
191
+ return node unless node.type == :statements_node
192
+
193
+ comment_offset = comment.location.start_offset
194
+ node.body.find { |statement| statement.location.start_offset > comment_offset } || node.body.last || node
195
+ end
196
+
165
197
  def i18n_receiver?(receiver)
166
198
  case receiver.type
167
199
  when :constant_read_node
168
200
  receiver.name == :I18n
169
201
  when :constant_path_node
170
- receiver.parent.nil? && receiver.child&.name == :I18n
202
+ receiver.parent.nil? && receiver.name == :I18n
171
203
  else
172
204
  false
173
205
  end
@@ -280,7 +312,7 @@ module I18n::Tasks::Scanners::PrismScanners
280
312
 
281
313
  # We need to check for `node.receiver` since node is the `human` call
282
314
  model_name = if current_class.present? && rails_model_method_called_on_current_class?(node.receiver)
283
- current_class.path.flatten.map!(&:underscore).join(".")
315
+ current_class.path.flatten.map!(&:underscore).join("/")
284
316
  elsif node.receiver&.receiver&.type == :constant_read_node
285
317
  node.receiver&.receiver&.name&.to_s&.underscore
286
318
  end
@@ -312,7 +344,14 @@ module I18n::Tasks::Scanners::PrismScanners
312
344
  end
313
345
 
314
346
  def rails_model_method_called_on_current_class?(node)
315
- node.receiver.nil? || (node.receiver&.name&.to_s == "class" && node.receiver.receiver&.is_a?(Prism::SelfNode))
347
+ case node.receiver&.type
348
+ when :call_node
349
+ node.receiver.name == :class && node.receiver.receiver&.is_a?(Prism::SelfNode)
350
+ when nil
351
+ true
352
+ else
353
+ false
354
+ end
316
355
  end
317
356
 
318
357
  def rails_handle_human_attribute_name(node)
@@ -320,21 +359,36 @@ module I18n::Tasks::Scanners::PrismScanners
320
359
  # Arguments empty or cannot be processed, e.g. if it is a call
321
360
  return unless array_args.size == 1 && keywords.empty?
322
361
 
362
+ attribute_name = array_args.first
363
+ return unless attribute_name.is_a?(String)
364
+
365
+ # Convert dots to slashes to match Rails' human_attribute_name behavior
366
+ # Rails converts "status.active" in Product.human_attribute_name("status.active")
367
+ # to look for activerecord.attributes.product/status.active
368
+ # When attribute contains a dot, use slash separator between model and attribute
369
+ separator = attribute_name.include?(".") ? "/" : "."
370
+
323
371
  # Handle if called on `self.class` or if the current_class has `model_name.i18n_key`
324
372
  key = if current_class.present? && rails_model_method_called_on_current_class?(node)
325
373
  [
326
374
  :activerecord,
327
375
  :attributes,
328
- current_class.path.flatten.map!(&:underscore).join("."),
329
- array_args.first
330
- ].join(".")
376
+ current_class.path.flatten.map!(&:underscore).join("/")
377
+ ].join(".") + separator + attribute_name
331
378
  elsif node.receiver&.name.present?
379
+ # For namespaced models (Foo::Bar), Rails uses "/" as separator in i18n keys
380
+ # (e.g. activerecord.attributes.foo/bar.name). ConstantPathNode#full_name
381
+ # returns "Foo::Bar"; replace "::" with "/" to match that convention.
382
+ model_key = if node.receiver.is_a?(Prism::ConstantPathNode)
383
+ node.receiver.full_name.gsub(/^::/, "").gsub("::", "/").underscore
384
+ else
385
+ node.receiver.name.to_s.underscore
386
+ end
332
387
  [
333
388
  :activerecord,
334
389
  :attributes,
335
- node.receiver.name.to_s.underscore,
336
- array_args.first
337
- ].join(".")
390
+ model_key
391
+ ].join(".") + separator + attribute_name
338
392
  else
339
393
  return
340
394
  end
@@ -343,7 +397,7 @@ module I18n::Tasks::Scanners::PrismScanners
343
397
  TranslationCall.new(
344
398
  node: node,
345
399
  key: key,
346
- candidate_keys: Array([:attributes, array_args.first].join(".")),
400
+ candidate_keys: attribute_name.include?(".") ? [] : Array([:attributes, attribute_name].join(".")),
347
401
  receiver: nil,
348
402
  parent: parent,
349
403
  options: {}
@@ -15,13 +15,14 @@ module I18n
15
15
  return key unless key.start_with?(DOT)
16
16
  fail "roots argument is required" unless roots.present?
17
17
 
18
- normalized_path = File.expand_path(path)
19
- (root = path_root(normalized_path, roots)) ||
18
+ cwd = Dir.pwd
19
+ normalized_path = File.expand_path(path, cwd)
20
+ (root = path_root(normalized_path, roots, cwd)) ||
20
21
  fail(CommandError, "Cannot resolve relative key \"#{key}\".\n" \
21
22
  "Set search.relative_roots in config/i18n-tasks.yml (currently #{roots.inspect})")
22
23
  normalized_path.sub!(root, "")
23
24
 
24
- if (exclude_method_name_paths || []).map { |p| expand_path(p) }.include?(root)
25
+ if expand_paths(exclude_method_name_paths || [], cwd).include?(root)
25
26
  "#{prefix(normalized_path)}#{key}"
26
27
  else
27
28
  "#{prefix(normalized_path, calling_method: calling_method)}#{key}"
@@ -34,21 +35,30 @@ module I18n
34
35
 
35
36
  # Detect the appropriate relative path root
36
37
  # @param [String] path /full/path
37
- # @param [Array<String>] roots array of full paths
38
+ # @param [Array<String>] roots relative or absolute paths
39
+ # @param [String] cwd the working directory to expand the roots against
38
40
  # @return [String] the closest ancestor root for path, with a trailing {File::SEPARATOR}.
39
- def path_root(path, roots)
40
- roots.map do |p|
41
- expand_path(p)
42
- end.sort.reverse_each.detect do |root|
41
+ def path_root(path, roots, cwd)
42
+ expand_paths(roots, cwd).sort.reverse_each.detect do |root|
43
43
  path.start_with?(root)
44
44
  end
45
45
  end
46
46
 
47
+ # Expanding a relative path reads the working directory, and this runs for each relative key:
48
+ # the working directory is read once by the caller, and the paths expanded against it are kept.
49
+ # @param [Array<String>] paths relative or absolute paths
50
+ # @param [String] cwd the working directory to expand them against
51
+ # @return [Array<String>] the paths expanded with {#expand_path}
52
+ def expand_paths(paths, cwd)
53
+ (@relative_keys_expanded_paths ||= {})[[cwd, paths]] ||= paths.map { |path| expand_path(path, cwd) }
54
+ end
55
+
47
56
  # Expand a path and add a trailing {File::SEPARATOR}
48
- # @param [String] path relative path
57
+ # @param [String] path relative or absolute path
58
+ # @param [String] cwd the working directory to expand it against
49
59
  # @return [String] absolute path, with a trailing {File::SEPARATOR}.
50
- def expand_path(path)
51
- File.expand_path(path) + File::SEPARATOR
60
+ def expand_path(path, cwd)
61
+ File.expand_path(path, cwd) + File::SEPARATOR
52
62
  end
53
63
 
54
64
  # @param normalized_path [String] path/relative/to/a/root
@@ -9,6 +9,7 @@ require "i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher"
9
9
  require "i18n/tasks/scanners/ast_matchers/message_receivers_matcher"
10
10
  require "i18n/tasks/scanners/ast_matchers/rails_model_matcher"
11
11
  require "i18n/tasks/scanners/prism_scanners/visitor"
12
+ require "prism"
12
13
 
13
14
  module I18n::Tasks::Scanners
14
15
  # Scan for I18n.translate calls using whitequark/parser primarily and Prism if configured.
@@ -19,6 +20,11 @@ module I18n::Tasks::Scanners
19
20
  include ::I18n::Tasks::Logging
20
21
 
21
22
  MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/
23
+ # Only the magic comments need comments attached to nodes, and they all contain this.
24
+ MAGIC_COMMENT_MARKER = "i18n-tasks-"
25
+
26
+ # Prism 2.0 attaches comments to nodes while parsing, and removed ParseResult#attach_comments!
27
+ PRISM_PARSE_OPTIONS = Prism::ParseResult.method_defined?(:attach_comments!) ? {} : {attach_comments: true}
22
28
 
23
29
  protected
24
30
 
@@ -37,7 +43,7 @@ module I18n::Tasks::Scanners
37
43
 
38
44
  def ast_parser_parse_file(path)
39
45
  setup_ast_parser
40
- ast, comments = path_to_ast_and_comments(path)
46
+ ast, comments = @parse_cache.fetch([self.class, path]) { path_to_ast_and_comments(path) }
41
47
 
42
48
  ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
43
49
  end
@@ -149,6 +155,7 @@ module I18n::Tasks::Scanners
149
155
  AstMatchers::MessageReceiversMatcher.new(
150
156
  receivers: [
151
157
  AST::Node.new(:const, [nil, :I18n]),
158
+ AST::Node.new(:const, [AST::Node.new(:cbase), :I18n]),
152
159
  nil
153
160
  ],
154
161
  message: message,
@@ -167,14 +174,18 @@ module I18n::Tasks::Scanners
167
174
  # Extract all occurrences of translate calls from the file at the given path.
168
175
  # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
169
176
  def prism_parse_file(path)
170
- # Need File.expand_path for JRuby
171
- process_prism_results(path, Prism.parse_file(File.expand_path(path)))
177
+ source = File.read(path)
178
+ options = source.include?(MAGIC_COMMENT_MARKER) ? PRISM_PARSE_OPTIONS : {}
179
+ process_prism_results(path, Prism.parse(source, filepath: path, **options))
172
180
  end
173
181
 
174
182
  # This method handles only parsing to be able to test it properly.
175
183
  # Therefore it cannot handle the parsing itself.
176
184
  def process_prism_results(path, parse_results)
177
- comments = parse_results.attach_comments!
185
+ if parse_results.respond_to?(:attach_comments!) && parse_results.source.source.include?(MAGIC_COMMENT_MARKER)
186
+ parse_results.attach_comments!
187
+ end
188
+ comments = parse_results.comments
178
189
  parsed = parse_results.value
179
190
 
180
191
  # Check for magic comment to skip prism parsing, fallback to Parser AST
@@ -187,7 +198,29 @@ module I18n::Tasks::Scanners
187
198
  parsed.accept(visitor)
188
199
 
189
200
  occurrences = []
190
- visitor.process.each do |translation_call|
201
+ translation_calls = visitor.process
202
+
203
+ # Prism can leave some trailing comments unattached to AST nodes
204
+ # (for example, comments right before a method `end`).
205
+ comments.each do |comment|
206
+ next if visitor.processed_magic_comment_ids.include?(comment.object_id)
207
+
208
+ content = comment.respond_to?(:slice) ? comment.slice : comment.location.slice
209
+ next unless MAGIC_COMMENT_PREFIX.match?(content)
210
+
211
+ payload = content.sub(MAGIC_COMMENT_PREFIX, "").delete("#").strip
212
+ parsed_comment = Prism.parse(payload)
213
+ next if parsed_comment.respond_to?(:errors) && parsed_comment.errors.any?
214
+
215
+ comment_visitor = I18n::Tasks::Scanners::PrismScanners::Visitor.new
216
+ parsed_comment.value.accept(comment_visitor)
217
+
218
+ comment_visitor.process.each do |translation_call|
219
+ translation_calls << translation_call.with_node(comment)
220
+ end
221
+ end
222
+
223
+ translation_calls.each do |translation_call|
191
224
  result = translation_call.occurrences(path)
192
225
  next unless result
193
226
 
@@ -8,7 +8,8 @@ module I18n
8
8
  PARENTHESIS_PAIRS = %w({} [] () <>).freeze
9
9
  START_KEYS = PARENTHESIS_PAIRS.to_set { |pair| pair[0] }.freeze
10
10
  END_KEYS = PARENTHESIS_PAIRS.to_h { |pair| [pair[0], pair[1]] }.freeze
11
- private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS
11
+ START_KEYS_PATTERN = Regexp.union(START_KEYS.to_a)
12
+ private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS, :START_KEYS_PATTERN
12
13
 
13
14
  # split a key by dots (.)
14
15
  # dots inside braces or parenthesis are not split on
@@ -18,16 +19,17 @@ module I18n
18
19
  # split_key 'a.b.c', 2 # => ['a', 'b.c']
19
20
  def split_key(key, max = Float::INFINITY)
20
21
  return [key] if max == 1
22
+ return split_on_dots(key, max) unless key.match?(START_KEYS_PATTERN)
21
23
 
22
24
  parts = []
23
25
  current_parenthesis_end_char = nil
24
- part = ""
26
+ part = +""
25
27
  key.each_char.with_index do |char, index|
26
28
  if current_parenthesis_end_char
27
- part += char
29
+ part << char
28
30
  current_parenthesis_end_char = nil if char == current_parenthesis_end_char
29
31
  elsif START_KEYS.include?(char)
30
- part += char
32
+ part << char
31
33
  current_parenthesis_end_char = END_KEYS[char]
32
34
  elsif char == "."
33
35
  parts << part
@@ -36,9 +38,9 @@ module I18n
36
38
  parts << remaining unless remaining.empty?
37
39
  return parts
38
40
  end
39
- part = ""
41
+ part = +""
40
42
  else
41
- part += char
43
+ part << char
42
44
  end
43
45
  end
44
46
 
@@ -47,6 +49,14 @@ module I18n
47
49
  current_parenthesis_end_char ? parts.concat(part.split(".")) : parts << part
48
50
  end
49
51
 
52
+ # Most keys have no parenthesis: String#split gives the same result
53
+ # without allocating a new string for every character.
54
+ def split_on_dots(key, max)
55
+ parts = key.split(".", (max == Float::INFINITY) ? -1 : max)
56
+ parts.pop if parts.last == ""
57
+ parts
58
+ end
59
+
50
60
  def last_key_part(key)
51
61
  split_key(key).last
52
62
  end
@@ -21,21 +21,7 @@ module I18n::Tasks
21
21
 
22
22
  @progress_bar = ProgressBar.create(total: pairs.flatten.size, format: "%a <%B> %e %c/%C (%p%%)")
23
23
 
24
- begin
25
- translated = translate_pairs(pairs, to: root.key, from: from)
26
- rescue => e
27
- warn "Translation for locale #{root.key} failed: #{e.message}"
28
- # If translate_pairs raised, try to salvage any partial translations
29
- # by attempting to translate each slice individually and collecting successes.
30
- translated = []
31
- pairs.group_by { |k_v| @i18n_tasks.html_key? k_v[0], from }.each do |_is_html, list_slice|
32
- translated.concat(fetch_translations(list_slice, to: root.key, from: from))
33
- rescue => e2
34
- warn "Partial translation failed for locale #{root.key}: #{e2.message} - leaving keys untranslated"
35
- # leave the original list_slice untranslated
36
- translated.concat(list_slice)
37
- end
38
- end
24
+ translated = translate_pairs(pairs, to: root.key, from: from)
39
25
 
40
26
  result.merge! Data::Tree::Siblings.from_flat_pairs(translated)
41
27
  end
@@ -53,13 +39,15 @@ module I18n::Tasks
53
39
  # copy reference keys as is, instead of translating
54
40
  reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
55
41
  list -= reference_key_vals
56
- result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.map do |is_html, list_slice|
57
- fetch_translations(list_slice, opts.merge(is_html ? options_for_html : options_for_plain))
58
- rescue => e
59
- warn "Translation slice failed: #{e.message} - leaving slice untranslated"
60
- # Return the original untranslated slice so already completed translations are preserved
61
- list_slice
62
- end.reduce(:+) || []
42
+ result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.flat_map do |is_html, list_slice|
43
+ slice_options = opts.merge(is_html ? options_for_html : options_for_plain)
44
+ translation_batches(list_slice, slice_options).flat_map do |batch|
45
+ fetch_translations(batch, slice_options)
46
+ rescue => e
47
+ warn "Translation batch failed: #{e.message} - skipping batch"
48
+ []
49
+ end
50
+ end
63
51
  result.concat(reference_key_vals)
64
52
  result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
65
53
  result
@@ -69,7 +57,13 @@ module I18n::Tasks
69
57
  # @return [Array<[String, Object]>] translated list
70
58
  def fetch_translations(list, opts)
71
59
  options = options_for_translate_values(**opts)
72
- from_values(list, translate_values(to_values(list, options), **options), options).tap do |result|
60
+ values = to_values(list, options)
61
+ translated_values = translate_values(values, **options)
62
+ if translated_values.size != values.size
63
+ fail CommandError, "Expected #{values.size} translations, got #{translated_values.size}"
64
+ end
65
+
66
+ from_values(list, translated_values, options).tap do |result|
73
67
  fail CommandError, no_results_error_message if result.blank?
74
68
  end
75
69
  end
@@ -188,6 +182,14 @@ module I18n::Tasks
188
182
  def options_for_plain
189
183
  end
190
184
 
185
+ # @param [Array<[String, Object]>] list
186
+ # @param [Hash] options
187
+ # @return [Array<Array<[String, Object]>>]
188
+ # @abstract
189
+ def translation_batches(list, _options)
190
+ [list]
191
+ end
192
+
191
193
  # @return [String]
192
194
  # @abstract
193
195
  def no_results_error_message
@@ -22,22 +22,21 @@ module I18n::Tasks::Translators
22
22
  protected
23
23
 
24
24
  def translate_values(list, from:, to:, **options)
25
- results = []
26
- list.each_slice(BATCH_SIZE) do |parts|
27
- res = DeepL.translate(
28
- parts,
29
- to_deepl_source_locale(from),
30
- to_deepl_target_locale(to),
31
- options_with_glossary(options, from, to)
32
- )
33
- if res.is_a?(DeepL::Resources::Text)
34
- results << res.text
35
- else
36
- results += res.map(&:text)
37
- end
38
-
39
- @progress_bar.progress += parts.size
25
+ return [] if list.empty?
26
+
27
+ res = DeepL.translate(
28
+ list,
29
+ to_deepl_source_locale(from),
30
+ to_deepl_target_locale(to),
31
+ options_with_glossary(options, from, to)
32
+ )
33
+ results = if res.is_a?(DeepL::Resources::Text)
34
+ [res.text]
35
+ else
36
+ res.map(&:text)
40
37
  end
38
+
39
+ @progress_bar.progress += list.size
41
40
  results
42
41
  end
43
42
 
@@ -55,6 +54,10 @@ module I18n::Tasks::Translators
55
54
  {preserve_formatting: true, tag_handling: "xml", html_escape: true}
56
55
  end
57
56
 
57
+ def translation_batches(list, _options)
58
+ list.each_slice(BATCH_SIZE).to_a
59
+ end
60
+
58
61
  # @param [String] value
59
62
  # @return [String] 'hello, %{name}' => 'hello, <i18n>%{name}</i18n>'
60
63
  def replace_interpolations(value)