i18n-tasks 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +321 -242
- data/Rakefile +5 -1
- data/benchmarks/bench_helper.rb +156 -0
- data/benchmarks/data_bench.rb +173 -0
- data/benchmarks/end_to_end_bench.rb +138 -0
- data/benchmarks/fixtures/generator.rb +257 -0
- data/benchmarks/results/baseline.json +250 -0
- data/benchmarks/run_all.rb +60 -0
- data/benchmarks/scanning_bench.rb +135 -0
- data/benchmarks/tree_bench.rb +205 -0
- data/config/locales/en.yml +10 -0
- data/config/locales/ru.yml +14 -0
- data/i18n-tasks.gemspec +1 -1
- data/lib/i18n/tasks/base_task.rb +2 -0
- data/lib/i18n/tasks/command/commands/check_prism.rb +67 -15
- data/lib/i18n/tasks/command/commands/prune.rb +57 -0
- data/lib/i18n/tasks/commands.rb +2 -0
- data/lib/i18n/tasks/data/adapter/json_adapter.rb +1 -1
- data/lib/i18n/tasks/data/file_formats.rb +4 -1
- data/lib/i18n/tasks/data/router/conservative_router.rb +13 -22
- data/lib/i18n/tasks/data/router/pattern_router.rb +26 -14
- data/lib/i18n/tasks/data/tree/node.rb +3 -3
- data/lib/i18n/tasks/data/tree/siblings.rb +11 -3
- data/lib/i18n/tasks/data/tree/traversal.rb +20 -14
- data/lib/i18n/tasks/interpolations.rb +61 -3
- data/lib/i18n/tasks/logging.rb +1 -2
- data/lib/i18n/tasks/missing_keys.rb +7 -38
- data/lib/i18n/tasks/plural_keys.rb +56 -0
- data/lib/i18n/tasks/prune.rb +18 -0
- data/lib/i18n/tasks/references.rb +9 -1
- data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +11 -2
- data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +16 -2
- data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +6 -3
- data/lib/i18n/tasks/scanners/file_scanner.rb +4 -1
- data/lib/i18n/tasks/scanners/files/file_finder.rb +29 -6
- data/lib/i18n/tasks/scanners/files/parse_cache.rb +24 -0
- data/lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb +33 -0
- data/lib/i18n/tasks/scanners/prism_scanners/nodes.rb +57 -23
- data/lib/i18n/tasks/scanners/prism_scanners/visitor.rb +70 -16
- data/lib/i18n/tasks/scanners/relative_keys.rb +21 -11
- data/lib/i18n/tasks/scanners/ruby_scanner.rb +37 -5
- data/lib/i18n/tasks/split_key.rb +16 -6
- data/lib/i18n/tasks/translators/base_translator.rb +25 -23
- data/lib/i18n/tasks/translators/deepl_translator.rb +18 -15
- data/lib/i18n/tasks/translators/google_translator.rb +41 -14
- data/lib/i18n/tasks/translators/openai_translator.rb +31 -17
- data/lib/i18n/tasks/translators/watsonx_translator.rb +8 -10
- data/lib/i18n/tasks/translators/yandex_translator.rb +4 -0
- data/lib/i18n/tasks/unused_keys.rb +35 -5
- data/lib/i18n/tasks/used_keys.rb +6 -1
- data/lib/i18n/tasks/version.rb +1 -1
- data/script/report_test_results.rb +58 -0
- data/templates/config/i18n-tasks.yml +11 -3
- metadata +17 -7
|
@@ -90,15 +90,14 @@ module I18n::Tasks
|
|
|
90
90
|
|
|
91
91
|
# Select the nodes for which the block returns true. Pre-order traversal.
|
|
92
92
|
# @return [Siblings] a new tree
|
|
93
|
-
def select_nodes(&block)
|
|
94
|
-
tree = Siblings.new
|
|
93
|
+
def select_nodes(parent: nil, &block)
|
|
94
|
+
tree = Siblings.new(parent: parent)
|
|
95
95
|
each do |node|
|
|
96
96
|
next unless block.yield(node)
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
)
|
|
98
|
+
copy = node.derive(parent: parent, children: nil)
|
|
99
|
+
copy.children = node.children.select_nodes(parent: copy, &block) if node.children
|
|
100
|
+
tree.append! copy
|
|
102
101
|
end
|
|
103
102
|
tree
|
|
104
103
|
end
|
|
@@ -167,34 +166,41 @@ module I18n::Tasks
|
|
|
167
166
|
end
|
|
168
167
|
|
|
169
168
|
def set_each_value!(val_pattern, key_pattern = nil, &value_proc) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
169
|
+
# A lone placeholder keeps the value's type, so that an array stays an array
|
|
170
|
+
lone_placeholder = val_pattern&.match(/\A%\{(\w+)\}\z/)&.[](1)&.to_sym
|
|
171
|
+
interpolate = lambda do |values|
|
|
172
|
+
lone_value = values[lone_placeholder] if lone_placeholder
|
|
173
|
+
return lone_value.dup unless lone_value.nil?
|
|
174
|
+
|
|
175
|
+
StringInterpolation.interpolate_soft(val_pattern, values)
|
|
176
|
+
end
|
|
170
177
|
value_proc ||= proc do |node|
|
|
171
178
|
node_value = node.value
|
|
172
179
|
next node_value if node.reference?
|
|
173
180
|
|
|
181
|
+
kept_value = node_value.is_a?(String) ? node_value.presence : node_value
|
|
174
182
|
human_key = ActiveSupport::Inflector.humanize(node.key.to_s)
|
|
175
183
|
full_key = node.full_key
|
|
176
184
|
default = (node.data[:occurrences] || []).detect { |o| o.default_arg.presence }.try(:default_arg)
|
|
177
185
|
if default.is_a?(Hash)
|
|
178
186
|
default.each_with_object({}) do |(k, v), h|
|
|
179
|
-
h[k] =
|
|
180
|
-
val_pattern,
|
|
187
|
+
h[k] = interpolate.call(
|
|
181
188
|
value: node_value,
|
|
182
189
|
human_key: human_key,
|
|
183
190
|
key: full_key,
|
|
184
191
|
default: v,
|
|
185
|
-
value_or_human_key:
|
|
186
|
-
value_or_default_or_human_key:
|
|
192
|
+
value_or_human_key: kept_value.nil? ? human_key : kept_value,
|
|
193
|
+
value_or_default_or_human_key: kept_value.nil? ? v || human_key : kept_value
|
|
187
194
|
)
|
|
188
195
|
end
|
|
189
196
|
else
|
|
190
|
-
|
|
191
|
-
val_pattern,
|
|
197
|
+
interpolate.call(
|
|
192
198
|
value: node_value,
|
|
193
199
|
human_key: human_key,
|
|
194
200
|
key: full_key,
|
|
195
201
|
default: default,
|
|
196
|
-
value_or_human_key:
|
|
197
|
-
value_or_default_or_human_key:
|
|
202
|
+
value_or_human_key: kept_value.nil? ? human_key : kept_value,
|
|
203
|
+
value_or_default_or_human_key: kept_value.nil? ? default || human_key : kept_value
|
|
198
204
|
)
|
|
199
205
|
end
|
|
200
206
|
end
|
|
@@ -11,17 +11,34 @@ module I18n::Tasks
|
|
|
11
11
|
locales ||= self.locales
|
|
12
12
|
base_locale ||= self.base_locale
|
|
13
13
|
result = empty_forest
|
|
14
|
-
|
|
14
|
+
plural_keys = Set.new
|
|
15
|
+
plural_nodes(data[base_locale]) do |base_node|
|
|
16
|
+
next unless base_node.parent?
|
|
17
|
+
|
|
18
|
+
group_key = base_node.full_key(root: false)
|
|
19
|
+
base_node.children.each { |form| plural_keys << "#{group_key}.#{form.key}" }
|
|
20
|
+
next if ignore_key?(group_key, :inconsistent_interpolations)
|
|
21
|
+
|
|
22
|
+
(locales - [base_locale]).each do |current_locale|
|
|
23
|
+
node = data[current_locale].first.children[group_key]
|
|
24
|
+
inconsistent_plural_forms(current_locale, base_node, node).each do |form|
|
|
25
|
+
next if ignore_key?("#{group_key}.#{form.key}", :inconsistent_interpolations)
|
|
26
|
+
|
|
27
|
+
result.merge!(form.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
15
31
|
|
|
16
32
|
data[base_locale].key_values.each do |key, value|
|
|
33
|
+
next if plural_keys.include?(key)
|
|
17
34
|
next if !value.is_a?(String) || ignore_key?(key, :inconsistent_interpolations)
|
|
18
35
|
|
|
19
|
-
base_vars =
|
|
36
|
+
base_vars = interpolation_variables(value)
|
|
20
37
|
(locales - [base_locale]).each do |current_locale|
|
|
21
38
|
node = data[current_locale].first.children[key]
|
|
22
39
|
next unless node&.value.is_a?(String)
|
|
23
40
|
|
|
24
|
-
if base_vars !=
|
|
41
|
+
if base_vars != interpolation_variables(node.value)
|
|
25
42
|
result.merge!(node.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
|
|
26
43
|
end
|
|
27
44
|
end
|
|
@@ -31,6 +48,47 @@ module I18n::Tasks
|
|
|
31
48
|
result
|
|
32
49
|
end
|
|
33
50
|
|
|
51
|
+
COUNT_VARIABLE = "%{count}"
|
|
52
|
+
# The forms that are used for at most one count when the locale has no plural rule in rails-i18n.
|
|
53
|
+
SINGLE_COUNT_PLURAL_FORMS = %w[zero one two 0 1].freeze
|
|
54
|
+
|
|
55
|
+
# Plural forms are compared as a group, because a locale has different forms than the base locale, and its forms
|
|
56
|
+
# cover different counts: the Russian `one` is used for 1, 21, 31, ..., so it needs `%{count}` even when the
|
|
57
|
+
# English `one` spells the number out. A form may only use the variables of the base forms, and must keep the
|
|
58
|
+
# variables of the same base form (or of `other` if the base locale does not have it). It may drop `%{count}`
|
|
59
|
+
# if it is used for at most one count, and must have it if it is used for several. The counts each form is used
|
|
60
|
+
# for come from the locale's rule in rails-i18n.
|
|
61
|
+
#
|
|
62
|
+
# Keys that are not plural forms in the translated group have no base form to compare with, and are skipped.
|
|
63
|
+
#
|
|
64
|
+
# @return [Array<I18n::Tasks::Data::Tree::Node>] the inconsistent forms of `node`
|
|
65
|
+
def inconsistent_plural_forms(locale, base_node, node)
|
|
66
|
+
return [] unless node&.children?
|
|
67
|
+
|
|
68
|
+
base_forms = base_node.children.to_h { |form| [form.key, interpolation_variables(form.value)] }
|
|
69
|
+
base_vars = base_forms.values.reduce(Set.new, :|)
|
|
70
|
+
forms = node.children.select { |form| form.leaf? && plural_suffix?(form.key) && form.value.is_a?(String) }
|
|
71
|
+
form_counts = plural_form_counts(locale, node.children.map(&:key))
|
|
72
|
+
forms.select do |form|
|
|
73
|
+
vars = interpolation_variables(form.value)
|
|
74
|
+
required = (base_forms[form.key] || base_forms["other"] || base_vars) - [COUNT_VARIABLE]
|
|
75
|
+
if several_plural_counts?(form_counts, form.key) && base_vars.include?(COUNT_VARIABLE)
|
|
76
|
+
required << COUNT_VARIABLE
|
|
77
|
+
end
|
|
78
|
+
!vars.subset?(base_vars) || !required.subset?(vars)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def several_plural_counts?(form_counts, form_key)
|
|
83
|
+
return !SINGLE_COUNT_PLURAL_FORMS.include?(form_key) if form_counts.nil?
|
|
84
|
+
|
|
85
|
+
form_counts.fetch(form_key, 0) > 1
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def interpolation_variables(value)
|
|
89
|
+
value.is_a?(String) ? Set.new(value.scan(I18n::Tasks::Interpolations.variable_regex)) : Set.new
|
|
90
|
+
end
|
|
91
|
+
|
|
34
92
|
def reserved_interpolations(locales: nil)
|
|
35
93
|
locales ||= self.locales
|
|
36
94
|
result = empty_forest
|
data/lib/i18n/tasks/logging.rb
CHANGED
|
@@ -25,8 +25,7 @@ module I18n::Tasks::Logging
|
|
|
25
25
|
def log_stderr(*args)
|
|
26
26
|
# We don't want output from different threads to get intermixed.
|
|
27
27
|
MUTEX.synchronize do
|
|
28
|
-
|
|
29
|
-
$stderr.puts(*args) # rubocop:disable Style/StderrPuts
|
|
28
|
+
warn(*args)
|
|
30
29
|
end
|
|
31
30
|
end
|
|
32
31
|
|
|
@@ -51,8 +51,9 @@ module I18n::Tasks
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
def missing_used_forest(locales, _base = base_locale)
|
|
54
|
+
used = used_tree(strict: true)
|
|
54
55
|
locales.inject(empty_forest) do |forest, locale|
|
|
55
|
-
forest.merge! missing_used_tree(locale)
|
|
56
|
+
forest.merge! missing_used_tree(locale, used: used)
|
|
56
57
|
end
|
|
57
58
|
end
|
|
58
59
|
|
|
@@ -83,13 +84,7 @@ module I18n::Tasks
|
|
|
83
84
|
@plural_keys_for_locale ||= {}
|
|
84
85
|
return @plural_keys_for_locale[locale] if @plural_keys_for_locale.key?(locale)
|
|
85
86
|
|
|
86
|
-
@plural_keys_for_locale[locale] =
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
# Loads rails-i18n pluralization config for the given locale.
|
|
90
|
-
def load_rails_i18n_pluralization!(locale)
|
|
91
|
-
path = File.join(Gem::Specification.find_by_name("rails-i18n").gem_dir, "rails", "pluralization", "#{locale}.rb")
|
|
92
|
-
eval(File.read(path), binding, path) # rubocop:disable Security/Eval
|
|
87
|
+
@plural_keys_for_locale[locale] = Set.new(rails_i18n_plural_config(locale)&.dig(:keys) || [])
|
|
93
88
|
end
|
|
94
89
|
|
|
95
90
|
# keys present in compared_to, but not in locale
|
|
@@ -107,8 +102,9 @@ module I18n::Tasks
|
|
|
107
102
|
end
|
|
108
103
|
|
|
109
104
|
# keys used in the code missing translations in locale
|
|
110
|
-
def missing_used_tree(locale)
|
|
111
|
-
used_tree(strict: true)
|
|
105
|
+
def missing_used_tree(locale, used: nil)
|
|
106
|
+
used ||= used_tree(strict: true)
|
|
107
|
+
used.select_keys do |key, node|
|
|
112
108
|
occurrences = node.data[:occurrences] || []
|
|
113
109
|
|
|
114
110
|
# An occurrence may carry candidate keys (for relative lookups). If any
|
|
@@ -138,7 +134,7 @@ module I18n::Tasks
|
|
|
138
134
|
end
|
|
139
135
|
|
|
140
136
|
def locale_key_missing?(locale, key)
|
|
141
|
-
!key_value?(key, locale) && !external_key?(key, locale) && !ignore_key?(key, :missing)
|
|
137
|
+
!key_value?(key, locale) && !external_key?(key, locale) && !ignore_key?(key, :missing, locale)
|
|
142
138
|
end
|
|
143
139
|
|
|
144
140
|
# @param [::I18n::Tasks::Data::Tree::Siblings] forest
|
|
@@ -170,32 +166,5 @@ module I18n::Tasks
|
|
|
170
166
|
end
|
|
171
167
|
forest
|
|
172
168
|
end
|
|
173
|
-
|
|
174
|
-
private
|
|
175
|
-
|
|
176
|
-
def plural_keys_for_locale(locale)
|
|
177
|
-
configuration = load_rails_i18n_pluralization!(locale)
|
|
178
|
-
if configuration[locale.to_sym].nil?
|
|
179
|
-
alternate_locale = alternate_locale_from(locale)
|
|
180
|
-
return Set.new if configuration[alternate_locale.to_sym].nil?
|
|
181
|
-
|
|
182
|
-
return set_from_rails_i18n_pluralization(configuration, alternate_locale)
|
|
183
|
-
end
|
|
184
|
-
set_from_rails_i18n_pluralization(configuration, locale)
|
|
185
|
-
rescue SystemCallError, IOError
|
|
186
|
-
Set.new
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def alternate_locale_from(locale)
|
|
190
|
-
re = /(\w{2})-*(\w{2,3})*/
|
|
191
|
-
match = locale.match(re)
|
|
192
|
-
language_code = match[1]
|
|
193
|
-
country_code = match[2]
|
|
194
|
-
"#{language_code}-#{country_code.upcase}"
|
|
195
|
-
end
|
|
196
|
-
|
|
197
|
-
def set_from_rails_i18n_pluralization(configuration, locale)
|
|
198
|
-
Set.new(configuration[locale.to_sym][:i18n][:plural][:keys])
|
|
199
|
-
end
|
|
200
169
|
end
|
|
201
170
|
end
|
|
@@ -60,4 +60,60 @@ module I18n::Tasks::PluralKeys
|
|
|
60
60
|
def plural_suffix?(key)
|
|
61
61
|
PLURAL_KEY_SUFFIXES.include?(key) || EXPLICIT_0_1.include?(key)
|
|
62
62
|
end
|
|
63
|
+
|
|
64
|
+
# @param [String] locale
|
|
65
|
+
# @return [Hash, nil] the rails-i18n plural configuration of the locale (`:keys` and `:rule`), nil if it has none.
|
|
66
|
+
def rails_i18n_plural_config(locale)
|
|
67
|
+
@rails_i18n_plural_configs ||= {}
|
|
68
|
+
return @rails_i18n_plural_configs[locale] if @rails_i18n_plural_configs.key?(locale)
|
|
69
|
+
|
|
70
|
+
@rails_i18n_plural_configs[locale] = begin
|
|
71
|
+
configuration = load_rails_i18n_pluralization!(locale)
|
|
72
|
+
configuration = configuration[locale.to_sym] || configuration[alternate_locale_from(locale).to_sym]
|
|
73
|
+
configuration&.dig(:i18n, :plural)
|
|
74
|
+
rescue SystemCallError, IOError
|
|
75
|
+
nil
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Loads rails-i18n pluralization config for the given locale.
|
|
80
|
+
def load_rails_i18n_pluralization!(locale)
|
|
81
|
+
path = File.join(Gem::Specification.find_by_name("rails-i18n").gem_dir, "rails", "pluralization", "#{locale}.rb")
|
|
82
|
+
eval(File.read(path), binding, path) # rubocop:disable Security/Eval
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Counts from 0 up to this cover every plural rule in rails-i18n, as they look at `n % 100` at most.
|
|
86
|
+
PLURAL_RULE_COUNTS = (0..1000)
|
|
87
|
+
|
|
88
|
+
# @param [String] locale
|
|
89
|
+
# @param [Array<String>] form_keys the plural forms of a group in the locale
|
|
90
|
+
# @return [Hash{String => Integer}, nil] how many counts in PLURAL_RULE_COUNTS select each form,
|
|
91
|
+
# with the locale's rails-i18n rule, nil if the locale has no rule.
|
|
92
|
+
def plural_form_counts(locale, form_keys)
|
|
93
|
+
rule = rails_i18n_plural_config(locale)&.dig(:rule)
|
|
94
|
+
return unless rule.respond_to?(:call)
|
|
95
|
+
|
|
96
|
+
@plural_form_counts ||= {}
|
|
97
|
+
@plural_form_counts[[locale, form_keys.sort]] ||=
|
|
98
|
+
PLURAL_RULE_COUNTS.map { |count| plural_form_for_count(rule, form_keys, count) }.tally
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
# The form I18n::Backend::Pluralization picks for the count.
|
|
104
|
+
def plural_form_for_count(rule, form_keys, count)
|
|
105
|
+
return "zero" if count.zero? && form_keys.include?("zero")
|
|
106
|
+
return count.to_s if count <= 1 && form_keys.include?(count.to_s)
|
|
107
|
+
|
|
108
|
+
category = rule.call(count).to_s
|
|
109
|
+
form_keys.include?(category) ? category : "other"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def alternate_locale_from(locale)
|
|
113
|
+
re = /(\w{2})-*(\w{2,3})*/
|
|
114
|
+
match = locale.match(re)
|
|
115
|
+
language_code = match[1]
|
|
116
|
+
country_code = match[2]
|
|
117
|
+
"#{language_code}-#{country_code&.upcase}"
|
|
118
|
+
end
|
|
63
119
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module I18n::Tasks::Prune
|
|
4
|
+
def prunable_keys(base_locale:, locales:)
|
|
5
|
+
diff_forest = empty_forest
|
|
6
|
+
locales.each do |locale|
|
|
7
|
+
# Based on the docs, missing_diff_tree only returns keys that are in the second argument but not in the first
|
|
8
|
+
locale_forest =
|
|
9
|
+
missing_diff_tree(base_locale, locale).tap do |locale_forest|
|
|
10
|
+
locale_forest.mv_key!(compile_key_pattern(base_locale), locale, root: true)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
diff_forest.merge!(locale_forest)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
diff_forest
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -7,7 +7,7 @@ module I18n::Tasks
|
|
|
7
7
|
# 2. Resolved references -- all the used references in their fully resolved form.
|
|
8
8
|
# 3. Reference keys -- all the used reference keys.
|
|
9
9
|
def process_references(usages,
|
|
10
|
-
data_refs = merge_reference_trees(
|
|
10
|
+
data_refs = merge_reference_trees(reference_trees))
|
|
11
11
|
fail ArgumentError, "usages must be a Data::Tree::Instance" unless usages.is_a?(Data::Tree::Siblings)
|
|
12
12
|
fail ArgumentError, "all_references must be a Data::Tree::Instance" unless data_refs.is_a?(Data::Tree::Siblings)
|
|
13
13
|
|
|
@@ -30,6 +30,14 @@ module I18n::Tasks
|
|
|
30
30
|
|
|
31
31
|
private
|
|
32
32
|
|
|
33
|
+
# Selecting the references of each locale before merging gives the same forest as
|
|
34
|
+
# data_forest.select_keys, without deep-copying every locale tree first.
|
|
35
|
+
def reference_trees
|
|
36
|
+
locales.inject(empty_forest) do |forest, locale|
|
|
37
|
+
forest.merge!(data[locale].select_keys { |_, node| node.reference? })
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
33
41
|
# @param [I18n::Tasks::Data::Tree::Node] ref
|
|
34
42
|
# @param [I18n::Tasks::Data::Tree::Node] usage
|
|
35
43
|
# @param [I18n::Tasks::Data::Tree::Siblings] raw_refs
|
|
@@ -12,7 +12,8 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
12
12
|
|
|
13
13
|
protected
|
|
14
14
|
|
|
15
|
-
# If the node type is of `%i(sym str int false true)`,
|
|
15
|
+
# If the node type is of `%i(sym str int false true)`, or a `dstr` of string literals only
|
|
16
|
+
# (see `adjacent_string_literals?`), return the value as a string.
|
|
16
17
|
# Otherwise, if `config[:strict]` is `false` and the type is of `%i(dstr dsym)`,
|
|
17
18
|
# return the source as if it were a string.
|
|
18
19
|
#
|
|
@@ -43,6 +44,8 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
43
44
|
# `nil` is returned when a dynamic value is encountered in strict mode. Propagate:
|
|
44
45
|
return nil if str.nil?
|
|
45
46
|
end
|
|
47
|
+
elsif adjacent_string_literals?(node)
|
|
48
|
+
node.children.map { |child| child.children[0] }.join
|
|
46
49
|
elsif !@scanner.config[:strict] && %i[dsym dstr].include?(node.type)
|
|
47
50
|
node.children.map do |child|
|
|
48
51
|
if %i[sym str].include?(child.type)
|
|
@@ -54,6 +57,12 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
54
57
|
end
|
|
55
58
|
end
|
|
56
59
|
|
|
60
|
+
# Adjacent string literals, e.g. `"a.b" \ ".c"`, and multi-line string literals are parsed
|
|
61
|
+
# as a `dstr` of `str` parts only: they are as static as a `str`.
|
|
62
|
+
def adjacent_string_literals?(node)
|
|
63
|
+
node.type == :dstr && node.children.any? && node.children.all? { |child| child.type == :str }
|
|
64
|
+
end
|
|
65
|
+
|
|
57
66
|
# Extract the whole hash from a node of type `:hash`
|
|
58
67
|
#
|
|
59
68
|
# @param node [AST::Node] a node of type `:hash`.
|
|
@@ -93,7 +102,7 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
93
102
|
# @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode.
|
|
94
103
|
def extract_array_as_string(node, array_join_with:, array_flatten: false, array_reject_blank: false)
|
|
95
104
|
children_strings = node.children.map do |child|
|
|
96
|
-
if %i[sym str int true false].include?(child.type)
|
|
105
|
+
if %i[sym str int true false].include?(child.type) || adjacent_string_literals?(child)
|
|
97
106
|
extract_string child
|
|
98
107
|
else
|
|
99
108
|
# ignore dynamic argument in strict mode
|
|
@@ -20,9 +20,23 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
20
20
|
|
|
21
21
|
value = children[2]
|
|
22
22
|
|
|
23
|
+
# Only process static string/symbol arguments; dynamic args (dstr/dsym) cannot be
|
|
24
|
+
# statically resolved and would produce a bogus key (e.g. "activerecord.attributes.product.").
|
|
25
|
+
return if value.nil?
|
|
26
|
+
return unless %i[str sym].include?(value.type) || adjacent_string_literals?(value)
|
|
27
|
+
|
|
23
28
|
model_name = underscore(receiver.to_a.last)
|
|
24
29
|
attribute = extract_string(value)
|
|
25
|
-
|
|
30
|
+
# Convert dots to slashes to match Rails' human_attribute_name behavior
|
|
31
|
+
# Rails converts "status.active" in Product.human_attribute_name("status.active")
|
|
32
|
+
# to look for activerecord.attributes.product/status.active
|
|
33
|
+
# The first dot becomes a slash: model_name.attribute -> model_name/first_part.rest
|
|
34
|
+
key = if attribute&.include?(".")
|
|
35
|
+
# Replace model_name.attribute with model_name/attribute for nested lookups
|
|
36
|
+
"activerecord.attributes.#{model_name}/#{attribute}"
|
|
37
|
+
else
|
|
38
|
+
"activerecord.attributes.#{model_name}.#{attribute}"
|
|
39
|
+
end
|
|
26
40
|
[
|
|
27
41
|
key,
|
|
28
42
|
I18n::Tasks::Scanners::Results::Occurrence.from_range(
|
|
@@ -61,7 +75,7 @@ module I18n::Tasks::Scanners::AstMatchers
|
|
|
61
75
|
end
|
|
62
76
|
|
|
63
77
|
def underscore(value)
|
|
64
|
-
value = value.dup
|
|
78
|
+
value = value.to_s.dup
|
|
65
79
|
value.gsub!(/(.)([A-Z])/, '\1_\2')
|
|
66
80
|
value.downcase!
|
|
67
81
|
end
|
|
@@ -83,7 +83,7 @@ module I18n::Tasks::Scanners
|
|
|
83
83
|
# Prism scanner, method called in RubyScanner
|
|
84
84
|
def prism_parse_file(path)
|
|
85
85
|
occurrences = []
|
|
86
|
-
content =
|
|
86
|
+
content = read_file(path)
|
|
87
87
|
|
|
88
88
|
content.scan(DEFAULT_REGEXP) do |indicator, code, _tailch, _rspace|
|
|
89
89
|
match = Regexp.last_match
|
|
@@ -104,7 +104,7 @@ module I18n::Tasks::Scanners
|
|
|
104
104
|
def process_code(path, code, content, start)
|
|
105
105
|
return [] if code.strip.empty? # skip empty ERB tags
|
|
106
106
|
|
|
107
|
-
process_prism_results(path, Prism.parse(code)).map do |key, occurrence|
|
|
107
|
+
process_prism_results(path, Prism.parse(code, **PRISM_PARSE_OPTIONS)).map do |key, occurrence|
|
|
108
108
|
[
|
|
109
109
|
key,
|
|
110
110
|
occurrence_from_position(
|
|
@@ -121,7 +121,10 @@ module I18n::Tasks::Scanners
|
|
|
121
121
|
def process_comments(path, code, content, start)
|
|
122
122
|
return [] if code.strip.empty?
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
# ERB comments are not executed, so only the `i18n-tasks-use` magic comments
|
|
125
|
+
# inside them are relevant. Turn every line into a Ruby comment so that
|
|
126
|
+
# commented-out code (e.g. `<%# t("key") %>`) is not reported as used.
|
|
127
|
+
parsed = Prism.parse(code.each_line.map { |line| "##{line.sub(/\A[ \t#]*/, "")}" }.join, **PRISM_PARSE_OPTIONS)
|
|
125
128
|
process_prism_results(path, parsed).map do |key, occurrence|
|
|
126
129
|
[
|
|
127
130
|
key,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "i18n/tasks/scanners/scanner"
|
|
4
|
+
require "i18n/tasks/scanners/files/parse_cache"
|
|
4
5
|
|
|
5
6
|
module I18n::Tasks::Scanners
|
|
6
7
|
# A base class for a scanner that analyses files.
|
|
@@ -13,11 +14,13 @@ module I18n::Tasks::Scanners
|
|
|
13
14
|
def initialize(
|
|
14
15
|
config: {},
|
|
15
16
|
file_finder_provider: Files::CachingFileFinderProvider.new,
|
|
16
|
-
file_reader: Files::CachingFileReader.new
|
|
17
|
+
file_reader: Files::CachingFileReader.new,
|
|
18
|
+
parse_cache: Files::ParseCache.new
|
|
17
19
|
)
|
|
18
20
|
super()
|
|
19
21
|
@config = config
|
|
20
22
|
@file_reader = file_reader
|
|
23
|
+
@parse_cache = parse_cache
|
|
21
24
|
@file_finder = file_finder_provider.get(**config.slice(:paths, :only, :exclude))
|
|
22
25
|
end
|
|
23
26
|
|
|
@@ -11,14 +11,14 @@ module I18n::Tasks::Scanners::Files
|
|
|
11
11
|
# absolute or relative to the working directory.
|
|
12
12
|
# @param only [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
|
|
13
13
|
# Files not matching any of the inclusion patterns will be excluded.
|
|
14
|
-
# @param exclude [
|
|
14
|
+
# @param exclude [Array<String>] {File.fnmatch}-compatible patterns of files to exclude.
|
|
15
15
|
# Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
|
|
16
16
|
def initialize(paths: ["."], only: nil, exclude: [])
|
|
17
17
|
fail "paths argument is required" if paths.nil?
|
|
18
18
|
|
|
19
19
|
@paths = paths
|
|
20
|
-
@include = only
|
|
21
|
-
@exclude = exclude || []
|
|
20
|
+
@include = only && split_extension_globs(only)
|
|
21
|
+
@exclude = split_extension_globs(exclude || [])
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
# Traverse the paths and yield the matching ones.
|
|
@@ -51,11 +51,34 @@ module I18n::Tasks::Scanners::Files
|
|
|
51
51
|
|
|
52
52
|
private
|
|
53
53
|
|
|
54
|
+
# Paths found under `.` start with `./`, which neither `*.rb` nor `app/**` match: try them without it too.
|
|
54
55
|
# @param path [String]
|
|
55
|
-
# @param
|
|
56
|
+
# @param split_globs [Array(Array<String>, Array<String>)] from {#split_extension_globs}
|
|
57
|
+
# @return [Boolean]
|
|
58
|
+
def path_fnmatch_any?(path, split_globs)
|
|
59
|
+
path_fnmatch_split_globs?(path, split_globs) ||
|
|
60
|
+
(path.start_with?("./") && path_fnmatch_split_globs?(path.delete_prefix("./"), split_globs))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# `File.fnmatch("*.png", path)` is `path.end_with?(".png")`, except for a path starting with a dot,
|
|
64
|
+
# which `*` does not match without `File::FNM_DOTMATCH`.
|
|
65
|
+
# @param path [String]
|
|
66
|
+
# @param split_globs [Array(Array<String>, Array<String>)] from {#split_extension_globs}
|
|
56
67
|
# @return [Boolean]
|
|
57
|
-
def
|
|
58
|
-
|
|
68
|
+
def path_fnmatch_split_globs?(path, split_globs)
|
|
69
|
+
suffixes, other_globs = split_globs
|
|
70
|
+
(!path.start_with?(".") && path.end_with?(*suffixes)) ||
|
|
71
|
+
other_globs.any? { |glob| File.fnmatch(glob, path) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# ASCII only: `String#end_with?` raises on incompatible encodings, where `File.fnmatch` returns false.
|
|
75
|
+
EXTENSION_GLOB = /\A\*(\.[\w-]+)+\z/
|
|
76
|
+
|
|
77
|
+
# @param globs [Array<String>]
|
|
78
|
+
# @return [Array(Array<String>, Array<String>)] the suffixes of the extension globs, and the other globs
|
|
79
|
+
def split_extension_globs(globs)
|
|
80
|
+
extension_globs, other_globs = globs.partition { |glob| EXTENSION_GLOB.match?(glob) }
|
|
81
|
+
[extension_globs.map { |glob| glob.delete_prefix("*") }, other_globs]
|
|
59
82
|
end
|
|
60
83
|
end
|
|
61
84
|
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "i18n/tasks/concurrent/cache"
|
|
4
|
+
|
|
5
|
+
module I18n::Tasks::Scanners::Files
|
|
6
|
+
# Keeps the files parsed while {#retain} runs, so that scanning them again
|
|
7
|
+
# (strictly, then non-strictly) does not parse them twice.
|
|
8
|
+
#
|
|
9
|
+
# @note This class is thread-safe.
|
|
10
|
+
class ParseCache
|
|
11
|
+
def retain
|
|
12
|
+
@cache = ::I18n::Tasks::Concurrent::Cache.new
|
|
13
|
+
yield
|
|
14
|
+
ensure
|
|
15
|
+
@cache = nil
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# @param key [Object] identifies the parse, e.g. the scanner class and the path.
|
|
19
|
+
# @yieldreturn the parse, computed only if not retained.
|
|
20
|
+
def fetch(key, &)
|
|
21
|
+
@cache ? @cache.fetch(key, &) : yield
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -16,11 +16,27 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
16
16
|
end
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
def visit_local_variable_read_node(node)
|
|
20
|
+
# Keep this node to know that we cannot resolve this argument statically
|
|
21
|
+
node
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def visit_constant_read_node(node)
|
|
25
|
+
# Keep this node to know that we cannot resolve this argument statically
|
|
26
|
+
node
|
|
27
|
+
end
|
|
28
|
+
|
|
19
29
|
# Cannot handle arguments that are calls
|
|
20
30
|
def visit_call_node(_node)
|
|
21
31
|
nil
|
|
22
32
|
end
|
|
23
33
|
|
|
34
|
+
# Shorthand hash syntax (e.g. `scope:` as shorthand for `scope: scope`) wraps
|
|
35
|
+
# the implicit value in an ImplicitNode — delegate to the inner node.
|
|
36
|
+
def visit_implicit_node(node)
|
|
37
|
+
visit(node.value)
|
|
38
|
+
end
|
|
39
|
+
|
|
24
40
|
def visit_symbol_node(node)
|
|
25
41
|
node.value
|
|
26
42
|
end
|
|
@@ -29,6 +45,23 @@ module I18n::Tasks::Scanners::PrismScanners
|
|
|
29
45
|
node.content
|
|
30
46
|
end
|
|
31
47
|
|
|
48
|
+
# Interpolated nodes (e.g. `Product.human_attribute_name("status.#{status}")`) contain dynamic content that cannot
|
|
49
|
+
# be statically resolved: return nil. Adjacent string literals (e.g. `"a.b" \ ".c"`) are parsed as an
|
|
50
|
+
# interpolated node too, but are static.
|
|
51
|
+
def visit_interpolated_string_node(node)
|
|
52
|
+
return unless node.parts.all?(Prism::StringNode)
|
|
53
|
+
|
|
54
|
+
node.parts.map(&:content).join
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def visit_interpolated_symbol_node(_node)
|
|
58
|
+
nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def visit_interpolated_x_string_node(_node)
|
|
62
|
+
nil
|
|
63
|
+
end
|
|
64
|
+
|
|
32
65
|
def visit_array_node(node)
|
|
33
66
|
node.child_nodes.map { |child| visit(child) }
|
|
34
67
|
end
|