i18n-tasks 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +321 -242
- data/Rakefile +5 -1
- data/benchmarks/bench_helper.rb +156 -0
- data/benchmarks/data_bench.rb +173 -0
- data/benchmarks/end_to_end_bench.rb +138 -0
- data/benchmarks/fixtures/generator.rb +257 -0
- data/benchmarks/results/baseline.json +250 -0
- data/benchmarks/run_all.rb +60 -0
- data/benchmarks/scanning_bench.rb +135 -0
- data/benchmarks/tree_bench.rb +205 -0
- data/config/locales/en.yml +10 -0
- data/config/locales/ru.yml +14 -0
- data/i18n-tasks.gemspec +1 -1
- data/lib/i18n/tasks/base_task.rb +2 -0
- data/lib/i18n/tasks/command/commands/check_prism.rb +67 -15
- data/lib/i18n/tasks/command/commands/prune.rb +57 -0
- data/lib/i18n/tasks/commands.rb +2 -0
- data/lib/i18n/tasks/data/adapter/json_adapter.rb +1 -1
- data/lib/i18n/tasks/data/file_formats.rb +4 -1
- data/lib/i18n/tasks/data/router/conservative_router.rb +13 -22
- data/lib/i18n/tasks/data/router/pattern_router.rb +26 -14
- data/lib/i18n/tasks/data/tree/node.rb +3 -3
- data/lib/i18n/tasks/data/tree/siblings.rb +11 -3
- data/lib/i18n/tasks/data/tree/traversal.rb +20 -14
- data/lib/i18n/tasks/interpolations.rb +61 -3
- data/lib/i18n/tasks/logging.rb +1 -2
- data/lib/i18n/tasks/missing_keys.rb +7 -38
- data/lib/i18n/tasks/plural_keys.rb +56 -0
- data/lib/i18n/tasks/prune.rb +18 -0
- data/lib/i18n/tasks/references.rb +9 -1
- data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +11 -2
- data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +16 -2
- data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +7 -3
- data/lib/i18n/tasks/scanners/file_scanner.rb +4 -1
- data/lib/i18n/tasks/scanners/files/file_finder.rb +29 -6
- data/lib/i18n/tasks/scanners/files/parse_cache.rb +24 -0
- data/lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb +33 -0
- data/lib/i18n/tasks/scanners/prism_scanners/nodes.rb +57 -23
- data/lib/i18n/tasks/scanners/prism_scanners/visitor.rb +70 -16
- data/lib/i18n/tasks/scanners/relative_keys.rb +21 -11
- data/lib/i18n/tasks/scanners/ruby_scanner.rb +38 -5
- data/lib/i18n/tasks/split_key.rb +16 -6
- data/lib/i18n/tasks/translators/base_translator.rb +25 -23
- data/lib/i18n/tasks/translators/deepl_translator.rb +18 -15
- data/lib/i18n/tasks/translators/google_translator.rb +41 -14
- data/lib/i18n/tasks/translators/openai_translator.rb +31 -17
- data/lib/i18n/tasks/translators/watsonx_translator.rb +8 -10
- data/lib/i18n/tasks/translators/yandex_translator.rb +4 -0
- data/lib/i18n/tasks/unused_keys.rb +35 -5
- data/lib/i18n/tasks/used_keys.rb +6 -1
- data/lib/i18n/tasks/version.rb +1 -1
- data/script/report_test_results.rb +58 -0
- data/templates/config/i18n-tasks.yml +11 -3
- metadata +17 -7
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# Used in the PrismScanners::Visitor class.
|
|
5
5
|
module I18n::Tasks::Scanners::PrismScanners
|
|
6
6
|
class Root
|
|
7
|
-
attr_reader(:calls, :translation_calls, :children, :node, :parent, :rails
|
|
7
|
+
attr_reader(:calls, :translation_calls, :children, :node, :parent, :rails)
|
|
8
8
|
|
|
9
9
|
def initialize(node: nil, parent: nil, file_path: nil, rails: false)
|
|
10
10
|
@calls = []
|
|
@@ -16,6 +16,10 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
16
16
|
@file_path = file_path
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
def file_path
|
|
20
|
+
@file_path || @parent&.file_path
|
|
21
|
+
end
|
|
22
|
+
|
|
19
23
|
def add_child(node)
|
|
20
24
|
@children << node
|
|
21
25
|
node
|
|
@@ -30,15 +34,19 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
30
34
|
end
|
|
31
35
|
|
|
32
36
|
def rails_view?
|
|
33
|
-
rails && file_path.present?
|
|
34
|
-
end
|
|
37
|
+
return false unless rails && file_path.present?
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
if file_path.include?("app/views/")
|
|
40
|
+
true
|
|
41
|
+
elsif file_path.include?("app/components/")
|
|
42
|
+
!file_path.end_with?(".rb")
|
|
43
|
+
else
|
|
44
|
+
false
|
|
45
|
+
end
|
|
38
46
|
end
|
|
39
47
|
|
|
40
48
|
def support_relative_keys?
|
|
41
|
-
rails_view?
|
|
49
|
+
rails_view?
|
|
42
50
|
end
|
|
43
51
|
|
|
44
52
|
def support_candidate_keys?
|
|
@@ -47,8 +55,15 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
47
55
|
|
|
48
56
|
def path
|
|
49
57
|
if rails_view?
|
|
50
|
-
folder_path = file_path.
|
|
58
|
+
folder_path = if file_path.include?("app/views/")
|
|
59
|
+
file_path.sub(%r{app/views/}, "").split("/")
|
|
60
|
+
else
|
|
61
|
+
file_path.sub(%r{app/components/}, "").split("/")
|
|
62
|
+
end
|
|
63
|
+
|
|
51
64
|
name = folder_path.pop.split(".").first
|
|
65
|
+
# Remove leading underscores from partials
|
|
66
|
+
name = name[1..] if name.start_with?("_")
|
|
52
67
|
|
|
53
68
|
[*folder_path, name]
|
|
54
69
|
else
|
|
@@ -121,7 +136,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
121
136
|
# Example: parent.path = ["events", "create"], key = ".success"
|
|
122
137
|
# yields: ["events.create.success", "events.success"]
|
|
123
138
|
parent_path = parent&.path || []
|
|
124
|
-
rel_key = key[1..] # strip leading dot
|
|
139
|
+
rel_key = key[1..] # strip leading dot
|
|
125
140
|
|
|
126
141
|
candidates = []
|
|
127
142
|
parent_path_length = parent_path.length
|
|
@@ -134,15 +149,15 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
134
149
|
candidates.map { |c| c.gsub("..", ".") }
|
|
135
150
|
elsif relative_key?
|
|
136
151
|
# For relative keys in views, just append to the full path
|
|
137
|
-
[base_parts + parent.path + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
|
|
152
|
+
[base_parts + parent.path + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
|
|
138
153
|
elsif key.start_with?(".")
|
|
139
|
-
[base_parts + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
|
|
154
|
+
[base_parts + [key[1..]]].flatten.compact.join(".").gsub("..", ".")
|
|
140
155
|
elsif @candidate_keys.present?
|
|
141
156
|
([key] + @candidate_keys).map do |c|
|
|
142
|
-
[base_parts + [c]].flatten.compact.join(".").gsub("..", ".")
|
|
157
|
+
[base_parts + [c]].flatten.compact.join(".").gsub("..", ".")
|
|
143
158
|
end
|
|
144
159
|
else
|
|
145
|
-
[base_parts + [key]].flatten.compact.join(".").gsub("..", ".")
|
|
160
|
+
[base_parts + [key]].flatten.compact.join(".").gsub("..", ".")
|
|
146
161
|
end
|
|
147
162
|
end
|
|
148
163
|
|
|
@@ -150,26 +165,37 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
150
165
|
|
|
151
166
|
def scope
|
|
152
167
|
return nil if @options.nil?
|
|
153
|
-
return nil unless @options
|
|
168
|
+
return nil unless @options.key?("scope")
|
|
169
|
+
|
|
170
|
+
scope_value = @options["scope"]
|
|
171
|
+
scope_array = Array(scope_value)
|
|
154
172
|
|
|
155
|
-
|
|
173
|
+
# ArgumentsVisitor returns a Prism::LocalVariableReadNode for local variables.
|
|
174
|
+
# Scope cannot be resolved if array is empty
|
|
175
|
+
if scope_value.is_a?(Prism::LocalVariableReadNode) ||
|
|
176
|
+
scope_array.empty? ||
|
|
177
|
+
!scope_array.all? { |s| s.is_a?(String) || s.is_a?(Symbol) }
|
|
178
|
+
fail(ScopeError, "Could not process scope")
|
|
179
|
+
end
|
|
156
180
|
|
|
157
|
-
|
|
181
|
+
scope_array.join(".")
|
|
158
182
|
end
|
|
159
183
|
|
|
160
184
|
def occurrence(file_path)
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
location = local_node.location
|
|
185
|
+
location = @node.location
|
|
164
186
|
|
|
165
187
|
final = full_key
|
|
166
188
|
return nil if final.nil?
|
|
167
189
|
|
|
190
|
+
# A zero-length copy slices only the first line, not every line the node spans.
|
|
191
|
+
line = location.copy(length: 0).slice_lines.chomp
|
|
192
|
+
|
|
168
193
|
occurrence = ::I18n::Tasks::Scanners::Results::Occurrence.new(
|
|
169
194
|
path: file_path,
|
|
170
|
-
line:
|
|
171
|
-
pos: location.
|
|
172
|
-
|
|
195
|
+
line: line,
|
|
196
|
+
pos: location.start_character_offset,
|
|
197
|
+
# start_character_column counts the characters from the start of the file, twice.
|
|
198
|
+
line_pos: line.byteslice(0, location.start_column).length,
|
|
173
199
|
line_num: location.start_line,
|
|
174
200
|
raw_key: key,
|
|
175
201
|
candidate_keys: Array(final)
|
|
@@ -304,7 +330,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
304
330
|
end
|
|
305
331
|
|
|
306
332
|
def support_relative_keys?
|
|
307
|
-
controller? || mailer?
|
|
333
|
+
controller? || mailer? || view_component?
|
|
308
334
|
end
|
|
309
335
|
|
|
310
336
|
def support_candidate_keys?
|
|
@@ -323,6 +349,10 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
323
349
|
@rails && @node.name.to_s.end_with?("Mailer")
|
|
324
350
|
end
|
|
325
351
|
|
|
352
|
+
def view_component?
|
|
353
|
+
@rails && file_path.present? && file_path.include?("app/components/")
|
|
354
|
+
end
|
|
355
|
+
|
|
326
356
|
def path_name
|
|
327
357
|
path = @node.constant_path.full_name_parts.map { |s| s.to_s.underscore }
|
|
328
358
|
path.last.delete_suffix!("_controller") if controller?
|
|
@@ -345,7 +375,11 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
345
375
|
delegate(:support_candidate_keys?, to: :parent)
|
|
346
376
|
|
|
347
377
|
def path
|
|
348
|
-
|
|
378
|
+
if @parent.respond_to?(:view_component?) && @parent.view_component?
|
|
379
|
+
@parent.path
|
|
380
|
+
else
|
|
381
|
+
(@parent&.path || []) + [@node.name]
|
|
382
|
+
end
|
|
349
383
|
end
|
|
350
384
|
|
|
351
385
|
def name
|
|
@@ -14,7 +14,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
14
14
|
class Visitor < Prism::Visitor # rubocop:disable Metrics/ClassLength
|
|
15
15
|
MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/
|
|
16
16
|
|
|
17
|
-
attr_reader(:calls, :current_module, :current_class, :current_method, :root)
|
|
17
|
+
attr_reader(:calls, :current_module, :current_class, :current_method, :root, :processed_magic_comment_ids)
|
|
18
18
|
|
|
19
19
|
def initialize(rails: false, file_path: nil)
|
|
20
20
|
@calls = []
|
|
@@ -23,6 +23,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
23
23
|
@current_class = nil
|
|
24
24
|
@current_method = nil
|
|
25
25
|
@root = Root.new(file_path:, rails: rails)
|
|
26
|
+
@processed_magic_comment_ids = []
|
|
26
27
|
|
|
27
28
|
@rails = rails
|
|
28
29
|
|
|
@@ -118,6 +119,26 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
118
119
|
super
|
|
119
120
|
end
|
|
120
121
|
|
|
122
|
+
def visit_keyword_hash_node(node)
|
|
123
|
+
handle_comments(node)
|
|
124
|
+
super
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def visit_statements_node(node)
|
|
128
|
+
handle_comments(node)
|
|
129
|
+
super
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def visit_local_variable_write_node(node)
|
|
133
|
+
handle_comments(node)
|
|
134
|
+
super
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def visit_symbol_node(node)
|
|
138
|
+
handle_comments(node)
|
|
139
|
+
super
|
|
140
|
+
end
|
|
141
|
+
|
|
121
142
|
def process
|
|
122
143
|
@root.process
|
|
123
144
|
end
|
|
@@ -147,27 +168,38 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
147
168
|
|
|
148
169
|
next if match.nil?
|
|
149
170
|
|
|
171
|
+
@processed_magic_comment_ids << comment.object_id
|
|
172
|
+
|
|
150
173
|
string =
|
|
151
174
|
content.gsub(MAGIC_COMMENT_PREFIX, "").delete("#").strip
|
|
175
|
+
parse_result = Prism.parse(string)
|
|
176
|
+
next if parse_result.respond_to?(:errors) && parse_result.errors.any?
|
|
177
|
+
|
|
152
178
|
visitor = Visitor.new
|
|
153
|
-
|
|
154
|
-
.parse(string)
|
|
155
|
-
.value
|
|
156
|
-
.accept(visitor)
|
|
179
|
+
parse_result.value.accept(visitor)
|
|
157
180
|
|
|
158
181
|
# Process and remap the found translation calls to be for the found comment
|
|
159
182
|
visitor.process.each do |comment_node|
|
|
160
|
-
parent.add_translation_call(comment_node.with_node(node))
|
|
183
|
+
parent.add_translation_call(comment_node.with_parent(parent).with_node(commented_node(node, comment)))
|
|
161
184
|
end
|
|
162
185
|
end
|
|
163
186
|
end
|
|
164
187
|
|
|
188
|
+
# Prism 2.0 attaches a comment on its own line to the enclosing statements,
|
|
189
|
+
# Prism 1.x attached it to the next statement, or the previous one at the end.
|
|
190
|
+
def commented_node(node, comment)
|
|
191
|
+
return node unless node.type == :statements_node
|
|
192
|
+
|
|
193
|
+
comment_offset = comment.location.start_offset
|
|
194
|
+
node.body.find { |statement| statement.location.start_offset > comment_offset } || node.body.last || node
|
|
195
|
+
end
|
|
196
|
+
|
|
165
197
|
def i18n_receiver?(receiver)
|
|
166
198
|
case receiver.type
|
|
167
199
|
when :constant_read_node
|
|
168
200
|
receiver.name == :I18n
|
|
169
201
|
when :constant_path_node
|
|
170
|
-
receiver.parent.nil? && receiver.
|
|
202
|
+
receiver.parent.nil? && receiver.name == :I18n
|
|
171
203
|
else
|
|
172
204
|
false
|
|
173
205
|
end
|
|
@@ -280,7 +312,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
280
312
|
|
|
281
313
|
# We need to check for `node.receiver` since node is the `human` call
|
|
282
314
|
model_name = if current_class.present? && rails_model_method_called_on_current_class?(node.receiver)
|
|
283
|
-
current_class.path.flatten.map!(&:underscore).join("
|
|
315
|
+
current_class.path.flatten.map!(&:underscore).join("/")
|
|
284
316
|
elsif node.receiver&.receiver&.type == :constant_read_node
|
|
285
317
|
node.receiver&.receiver&.name&.to_s&.underscore
|
|
286
318
|
end
|
|
@@ -312,7 +344,14 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
312
344
|
end
|
|
313
345
|
|
|
314
346
|
def rails_model_method_called_on_current_class?(node)
|
|
315
|
-
|
|
347
|
+
case node.receiver&.type
|
|
348
|
+
when :call_node
|
|
349
|
+
node.receiver.name == :class && node.receiver.receiver&.is_a?(Prism::SelfNode)
|
|
350
|
+
when nil
|
|
351
|
+
true
|
|
352
|
+
else
|
|
353
|
+
false
|
|
354
|
+
end
|
|
316
355
|
end
|
|
317
356
|
|
|
318
357
|
def rails_handle_human_attribute_name(node)
|
|
@@ -320,21 +359,36 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
320
359
|
# Arguments empty or cannot be processed, e.g. if it is a call
|
|
321
360
|
return unless array_args.size == 1 && keywords.empty?
|
|
322
361
|
|
|
362
|
+
attribute_name = array_args.first
|
|
363
|
+
return unless attribute_name.is_a?(String)
|
|
364
|
+
|
|
365
|
+
# Convert dots to slashes to match Rails' human_attribute_name behavior
|
|
366
|
+
# Rails converts "status.active" in Product.human_attribute_name("status.active")
|
|
367
|
+
# to look for activerecord.attributes.product/status.active
|
|
368
|
+
# When attribute contains a dot, use slash separator between model and attribute
|
|
369
|
+
separator = attribute_name.include?(".") ? "/" : "."
|
|
370
|
+
|
|
323
371
|
# Handle if called on `self.class` or if the current_class has `model_name.i18n_key`
|
|
324
372
|
key = if current_class.present? && rails_model_method_called_on_current_class?(node)
|
|
325
373
|
[
|
|
326
374
|
:activerecord,
|
|
327
375
|
:attributes,
|
|
328
|
-
current_class.path.flatten.map!(&:underscore).join("
|
|
329
|
-
|
|
330
|
-
].join(".")
|
|
376
|
+
current_class.path.flatten.map!(&:underscore).join("/")
|
|
377
|
+
].join(".") + separator + attribute_name
|
|
331
378
|
elsif node.receiver&.name.present?
|
|
379
|
+
# For namespaced models (Foo::Bar), Rails uses "/" as separator in i18n keys
|
|
380
|
+
# (e.g. activerecord.attributes.foo/bar.name). ConstantPathNode#full_name
|
|
381
|
+
# returns "Foo::Bar"; replace "::" with "/" to match that convention.
|
|
382
|
+
model_key = if node.receiver.is_a?(Prism::ConstantPathNode)
|
|
383
|
+
node.receiver.full_name.gsub(/^::/, "").gsub("::", "/").underscore
|
|
384
|
+
else
|
|
385
|
+
node.receiver.name.to_s.underscore
|
|
386
|
+
end
|
|
332
387
|
[
|
|
333
388
|
:activerecord,
|
|
334
389
|
:attributes,
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
].join(".")
|
|
390
|
+
model_key
|
|
391
|
+
].join(".") + separator + attribute_name
|
|
338
392
|
else
|
|
339
393
|
return
|
|
340
394
|
end
|
|
@@ -343,7 +397,7 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
343
397
|
TranslationCall.new(
|
|
344
398
|
node: node,
|
|
345
399
|
key: key,
|
|
346
|
-
candidate_keys: Array([:attributes,
|
|
400
|
+
candidate_keys: attribute_name.include?(".") ? [] : Array([:attributes, attribute_name].join(".")),
|
|
347
401
|
receiver: nil,
|
|
348
402
|
parent: parent,
|
|
349
403
|
options: {}
|
|
@@ -15,13 +15,14 @@ module I18n
|
|
|
15
15
|
return key unless key.start_with?(DOT)
|
|
16
16
|
fail "roots argument is required" unless roots.present?
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
cwd = Dir.pwd
|
|
19
|
+
normalized_path = File.expand_path(path, cwd)
|
|
20
|
+
(root = path_root(normalized_path, roots, cwd)) ||
|
|
20
21
|
fail(CommandError, "Cannot resolve relative key \"#{key}\".\n" \
|
|
21
22
|
"Set search.relative_roots in config/i18n-tasks.yml (currently #{roots.inspect})")
|
|
22
23
|
normalized_path.sub!(root, "")
|
|
23
24
|
|
|
24
|
-
if (exclude_method_name_paths || []
|
|
25
|
+
if expand_paths(exclude_method_name_paths || [], cwd).include?(root)
|
|
25
26
|
"#{prefix(normalized_path)}#{key}"
|
|
26
27
|
else
|
|
27
28
|
"#{prefix(normalized_path, calling_method: calling_method)}#{key}"
|
|
@@ -34,21 +35,30 @@ module I18n
|
|
|
34
35
|
|
|
35
36
|
# Detect the appropriate relative path root
|
|
36
37
|
# @param [String] path /full/path
|
|
37
|
-
# @param [Array<String>] roots
|
|
38
|
+
# @param [Array<String>] roots relative or absolute paths
|
|
39
|
+
# @param [String] cwd the working directory to expand the roots against
|
|
38
40
|
# @return [String] the closest ancestor root for path, with a trailing {File::SEPARATOR}.
|
|
39
|
-
def path_root(path, roots)
|
|
40
|
-
roots.
|
|
41
|
-
expand_path(p)
|
|
42
|
-
end.sort.reverse_each.detect do |root|
|
|
41
|
+
def path_root(path, roots, cwd)
|
|
42
|
+
expand_paths(roots, cwd).sort.reverse_each.detect do |root|
|
|
43
43
|
path.start_with?(root)
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
# Expanding a relative path reads the working directory, and this runs for each relative key:
|
|
48
|
+
# the working directory is read once by the caller, and the paths expanded against it are kept.
|
|
49
|
+
# @param [Array<String>] paths relative or absolute paths
|
|
50
|
+
# @param [String] cwd the working directory to expand them against
|
|
51
|
+
# @return [Array<String>] the paths expanded with {#expand_path}
|
|
52
|
+
def expand_paths(paths, cwd)
|
|
53
|
+
(@relative_keys_expanded_paths ||= {})[[cwd, paths]] ||= paths.map { |path| expand_path(path, cwd) }
|
|
54
|
+
end
|
|
55
|
+
|
|
47
56
|
# Expand a path and add a trailing {File::SEPARATOR}
|
|
48
|
-
# @param [String] path relative path
|
|
57
|
+
# @param [String] path relative or absolute path
|
|
58
|
+
# @param [String] cwd the working directory to expand it against
|
|
49
59
|
# @return [String] absolute path, with a trailing {File::SEPARATOR}.
|
|
50
|
-
def expand_path(path)
|
|
51
|
-
File.expand_path(path) + File::SEPARATOR
|
|
60
|
+
def expand_path(path, cwd)
|
|
61
|
+
File.expand_path(path, cwd) + File::SEPARATOR
|
|
52
62
|
end
|
|
53
63
|
|
|
54
64
|
# @param normalized_path [String] path/relative/to/a/root
|
|
@@ -9,6 +9,7 @@ require "i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher"
|
|
|
9
9
|
require "i18n/tasks/scanners/ast_matchers/message_receivers_matcher"
|
|
10
10
|
require "i18n/tasks/scanners/ast_matchers/rails_model_matcher"
|
|
11
11
|
require "i18n/tasks/scanners/prism_scanners/visitor"
|
|
12
|
+
require "prism"
|
|
12
13
|
|
|
13
14
|
module I18n::Tasks::Scanners
|
|
14
15
|
# Scan for I18n.translate calls using whitequark/parser primarily and Prism if configured.
|
|
@@ -19,6 +20,11 @@ module I18n::Tasks::Scanners
|
|
|
19
20
|
include ::I18n::Tasks::Logging
|
|
20
21
|
|
|
21
22
|
MAGIC_COMMENT_PREFIX = /\A.\s*i18n-tasks-use\s+/
|
|
23
|
+
# Only the magic comments need comments attached to nodes, and they all contain this.
|
|
24
|
+
MAGIC_COMMENT_MARKER = "i18n-tasks-"
|
|
25
|
+
|
|
26
|
+
# Prism 2.0 attaches comments to nodes while parsing, and removed ParseResult#attach_comments!
|
|
27
|
+
PRISM_PARSE_OPTIONS = Prism::ParseResult.method_defined?(:attach_comments!) ? {} : {attach_comments: true}
|
|
22
28
|
|
|
23
29
|
protected
|
|
24
30
|
|
|
@@ -37,7 +43,7 @@ module I18n::Tasks::Scanners
|
|
|
37
43
|
|
|
38
44
|
def ast_parser_parse_file(path)
|
|
39
45
|
setup_ast_parser
|
|
40
|
-
ast, comments = path_to_ast_and_comments(path)
|
|
46
|
+
ast, comments = @parse_cache.fetch([self.class, path]) { path_to_ast_and_comments(path) }
|
|
41
47
|
|
|
42
48
|
ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
|
|
43
49
|
end
|
|
@@ -149,6 +155,7 @@ module I18n::Tasks::Scanners
|
|
|
149
155
|
AstMatchers::MessageReceiversMatcher.new(
|
|
150
156
|
receivers: [
|
|
151
157
|
AST::Node.new(:const, [nil, :I18n]),
|
|
158
|
+
AST::Node.new(:const, [AST::Node.new(:cbase), :I18n]),
|
|
152
159
|
nil
|
|
153
160
|
],
|
|
154
161
|
message: message,
|
|
@@ -167,14 +174,18 @@ module I18n::Tasks::Scanners
|
|
|
167
174
|
# Extract all occurrences of translate calls from the file at the given path.
|
|
168
175
|
# @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
|
|
169
176
|
def prism_parse_file(path)
|
|
170
|
-
|
|
171
|
-
|
|
177
|
+
source = File.read(path)
|
|
178
|
+
options = source.include?(MAGIC_COMMENT_MARKER) ? PRISM_PARSE_OPTIONS : {}
|
|
179
|
+
process_prism_results(path, Prism.parse(source, filepath: path, **options))
|
|
172
180
|
end
|
|
173
181
|
|
|
174
182
|
# This method handles only parsing to be able to test it properly.
|
|
175
183
|
# Therefore it cannot handle the parsing itself.
|
|
176
184
|
def process_prism_results(path, parse_results)
|
|
177
|
-
|
|
185
|
+
if parse_results.respond_to?(:attach_comments!) && parse_results.source.source.include?(MAGIC_COMMENT_MARKER)
|
|
186
|
+
parse_results.attach_comments!
|
|
187
|
+
end
|
|
188
|
+
comments = parse_results.comments
|
|
178
189
|
parsed = parse_results.value
|
|
179
190
|
|
|
180
191
|
# Check for magic comment to skip prism parsing, fallback to Parser AST
|
|
@@ -187,7 +198,29 @@ module I18n::Tasks::Scanners
|
|
|
187
198
|
parsed.accept(visitor)
|
|
188
199
|
|
|
189
200
|
occurrences = []
|
|
190
|
-
visitor.process
|
|
201
|
+
translation_calls = visitor.process
|
|
202
|
+
|
|
203
|
+
# Prism can leave some trailing comments unattached to AST nodes
|
|
204
|
+
# (for example, comments right before a method `end`).
|
|
205
|
+
comments.each do |comment|
|
|
206
|
+
next if visitor.processed_magic_comment_ids.include?(comment.object_id)
|
|
207
|
+
|
|
208
|
+
content = comment.respond_to?(:slice) ? comment.slice : comment.location.slice
|
|
209
|
+
next unless MAGIC_COMMENT_PREFIX.match?(content)
|
|
210
|
+
|
|
211
|
+
payload = content.sub(MAGIC_COMMENT_PREFIX, "").delete("#").strip
|
|
212
|
+
parsed_comment = Prism.parse(payload)
|
|
213
|
+
next if parsed_comment.respond_to?(:errors) && parsed_comment.errors.any?
|
|
214
|
+
|
|
215
|
+
comment_visitor = I18n::Tasks::Scanners::PrismScanners::Visitor.new
|
|
216
|
+
parsed_comment.value.accept(comment_visitor)
|
|
217
|
+
|
|
218
|
+
comment_visitor.process.each do |translation_call|
|
|
219
|
+
translation_calls << translation_call.with_node(comment)
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
translation_calls.each do |translation_call|
|
|
191
224
|
result = translation_call.occurrences(path)
|
|
192
225
|
next unless result
|
|
193
226
|
|
data/lib/i18n/tasks/split_key.rb
CHANGED
|
@@ -8,7 +8,8 @@ module I18n
|
|
|
8
8
|
PARENTHESIS_PAIRS = %w({} [] () <>).freeze
|
|
9
9
|
START_KEYS = PARENTHESIS_PAIRS.to_set { |pair| pair[0] }.freeze
|
|
10
10
|
END_KEYS = PARENTHESIS_PAIRS.to_h { |pair| [pair[0], pair[1]] }.freeze
|
|
11
|
-
|
|
11
|
+
START_KEYS_PATTERN = Regexp.union(START_KEYS.to_a)
|
|
12
|
+
private_constant :PARENTHESIS_PAIRS, :START_KEYS, :END_KEYS, :START_KEYS_PATTERN
|
|
12
13
|
|
|
13
14
|
# split a key by dots (.)
|
|
14
15
|
# dots inside braces or parenthesis are not split on
|
|
@@ -18,16 +19,17 @@ module I18n
|
|
|
18
19
|
# split_key 'a.b.c', 2 # => ['a', 'b.c']
|
|
19
20
|
def split_key(key, max = Float::INFINITY)
|
|
20
21
|
return [key] if max == 1
|
|
22
|
+
return split_on_dots(key, max) unless key.match?(START_KEYS_PATTERN)
|
|
21
23
|
|
|
22
24
|
parts = []
|
|
23
25
|
current_parenthesis_end_char = nil
|
|
24
|
-
part = ""
|
|
26
|
+
part = +""
|
|
25
27
|
key.each_char.with_index do |char, index|
|
|
26
28
|
if current_parenthesis_end_char
|
|
27
|
-
part
|
|
29
|
+
part << char
|
|
28
30
|
current_parenthesis_end_char = nil if char == current_parenthesis_end_char
|
|
29
31
|
elsif START_KEYS.include?(char)
|
|
30
|
-
part
|
|
32
|
+
part << char
|
|
31
33
|
current_parenthesis_end_char = END_KEYS[char]
|
|
32
34
|
elsif char == "."
|
|
33
35
|
parts << part
|
|
@@ -36,9 +38,9 @@ module I18n
|
|
|
36
38
|
parts << remaining unless remaining.empty?
|
|
37
39
|
return parts
|
|
38
40
|
end
|
|
39
|
-
part = ""
|
|
41
|
+
part = +""
|
|
40
42
|
else
|
|
41
|
-
part
|
|
43
|
+
part << char
|
|
42
44
|
end
|
|
43
45
|
end
|
|
44
46
|
|
|
@@ -47,6 +49,14 @@ module I18n
|
|
|
47
49
|
current_parenthesis_end_char ? parts.concat(part.split(".")) : parts << part
|
|
48
50
|
end
|
|
49
51
|
|
|
52
|
+
# Most keys have no parenthesis: String#split gives the same result
|
|
53
|
+
# without allocating a new string for every character.
|
|
54
|
+
def split_on_dots(key, max)
|
|
55
|
+
parts = key.split(".", (max == Float::INFINITY) ? -1 : max)
|
|
56
|
+
parts.pop if parts.last == ""
|
|
57
|
+
parts
|
|
58
|
+
end
|
|
59
|
+
|
|
50
60
|
def last_key_part(key)
|
|
51
61
|
split_key(key).last
|
|
52
62
|
end
|
|
@@ -21,21 +21,7 @@ module I18n::Tasks
|
|
|
21
21
|
|
|
22
22
|
@progress_bar = ProgressBar.create(total: pairs.flatten.size, format: "%a <%B> %e %c/%C (%p%%)")
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
translated = translate_pairs(pairs, to: root.key, from: from)
|
|
26
|
-
rescue => e
|
|
27
|
-
warn "Translation for locale #{root.key} failed: #{e.message}"
|
|
28
|
-
# If translate_pairs raised, try to salvage any partial translations
|
|
29
|
-
# by attempting to translate each slice individually and collecting successes.
|
|
30
|
-
translated = []
|
|
31
|
-
pairs.group_by { |k_v| @i18n_tasks.html_key? k_v[0], from }.each do |_is_html, list_slice|
|
|
32
|
-
translated.concat(fetch_translations(list_slice, to: root.key, from: from))
|
|
33
|
-
rescue => e2
|
|
34
|
-
warn "Partial translation failed for locale #{root.key}: #{e2.message} - leaving keys untranslated"
|
|
35
|
-
# leave the original list_slice untranslated
|
|
36
|
-
translated.concat(list_slice)
|
|
37
|
-
end
|
|
38
|
-
end
|
|
24
|
+
translated = translate_pairs(pairs, to: root.key, from: from)
|
|
39
25
|
|
|
40
26
|
result.merge! Data::Tree::Siblings.from_flat_pairs(translated)
|
|
41
27
|
end
|
|
@@ -53,13 +39,15 @@ module I18n::Tasks
|
|
|
53
39
|
# copy reference keys as is, instead of translating
|
|
54
40
|
reference_key_vals = list.select { |_k, v| v.is_a? Symbol } || []
|
|
55
41
|
list -= reference_key_vals
|
|
56
|
-
result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
42
|
+
result = list.group_by { |k_v| @i18n_tasks.html_key? k_v[0], opts[:from] }.flat_map do |is_html, list_slice|
|
|
43
|
+
slice_options = opts.merge(is_html ? options_for_html : options_for_plain)
|
|
44
|
+
translation_batches(list_slice, slice_options).flat_map do |batch|
|
|
45
|
+
fetch_translations(batch, slice_options)
|
|
46
|
+
rescue => e
|
|
47
|
+
warn "Translation batch failed: #{e.message} - skipping batch"
|
|
48
|
+
[]
|
|
49
|
+
end
|
|
50
|
+
end
|
|
63
51
|
result.concat(reference_key_vals)
|
|
64
52
|
result.sort! { |a, b| key_pos[a[0]] <=> key_pos[b[0]] }
|
|
65
53
|
result
|
|
@@ -69,7 +57,13 @@ module I18n::Tasks
|
|
|
69
57
|
# @return [Array<[String, Object]>] translated list
|
|
70
58
|
def fetch_translations(list, opts)
|
|
71
59
|
options = options_for_translate_values(**opts)
|
|
72
|
-
|
|
60
|
+
values = to_values(list, options)
|
|
61
|
+
translated_values = translate_values(values, **options)
|
|
62
|
+
if translated_values.size != values.size
|
|
63
|
+
fail CommandError, "Expected #{values.size} translations, got #{translated_values.size}"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
from_values(list, translated_values, options).tap do |result|
|
|
73
67
|
fail CommandError, no_results_error_message if result.blank?
|
|
74
68
|
end
|
|
75
69
|
end
|
|
@@ -188,6 +182,14 @@ module I18n::Tasks
|
|
|
188
182
|
def options_for_plain
|
|
189
183
|
end
|
|
190
184
|
|
|
185
|
+
# @param [Array<[String, Object]>] list
|
|
186
|
+
# @param [Hash] options
|
|
187
|
+
# @return [Array<Array<[String, Object]>>]
|
|
188
|
+
# @abstract
|
|
189
|
+
def translation_batches(list, _options)
|
|
190
|
+
[list]
|
|
191
|
+
end
|
|
192
|
+
|
|
191
193
|
# @return [String]
|
|
192
194
|
# @abstract
|
|
193
195
|
def no_results_error_message
|
|
@@ -22,22 +22,21 @@ module I18n::Tasks::Translators
|
|
|
22
22
|
protected
|
|
23
23
|
|
|
24
24
|
def translate_values(list, from:, to:, **options)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
@progress_bar.progress += parts.size
|
|
25
|
+
return [] if list.empty?
|
|
26
|
+
|
|
27
|
+
res = DeepL.translate(
|
|
28
|
+
list,
|
|
29
|
+
to_deepl_source_locale(from),
|
|
30
|
+
to_deepl_target_locale(to),
|
|
31
|
+
options_with_glossary(options, from, to)
|
|
32
|
+
)
|
|
33
|
+
results = if res.is_a?(DeepL::Resources::Text)
|
|
34
|
+
[res.text]
|
|
35
|
+
else
|
|
36
|
+
res.map(&:text)
|
|
40
37
|
end
|
|
38
|
+
|
|
39
|
+
@progress_bar.progress += list.size
|
|
41
40
|
results
|
|
42
41
|
end
|
|
43
42
|
|
|
@@ -55,6 +54,10 @@ module I18n::Tasks::Translators
|
|
|
55
54
|
{preserve_formatting: true, tag_handling: "xml", html_escape: true}
|
|
56
55
|
end
|
|
57
56
|
|
|
57
|
+
def translation_batches(list, _options)
|
|
58
|
+
list.each_slice(BATCH_SIZE).to_a
|
|
59
|
+
end
|
|
60
|
+
|
|
58
61
|
# @param [String] value
|
|
59
62
|
# @return [String] 'hello, %{name}' => 'hello, <i18n>%{name}</i18n>'
|
|
60
63
|
def replace_interpolations(value)
|