i18n-tasks 1.1.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +321 -242
  3. data/Rakefile +5 -1
  4. data/benchmarks/bench_helper.rb +156 -0
  5. data/benchmarks/data_bench.rb +173 -0
  6. data/benchmarks/end_to_end_bench.rb +138 -0
  7. data/benchmarks/fixtures/generator.rb +257 -0
  8. data/benchmarks/results/baseline.json +250 -0
  9. data/benchmarks/run_all.rb +60 -0
  10. data/benchmarks/scanning_bench.rb +135 -0
  11. data/benchmarks/tree_bench.rb +205 -0
  12. data/config/locales/en.yml +10 -0
  13. data/config/locales/ru.yml +14 -0
  14. data/i18n-tasks.gemspec +1 -1
  15. data/lib/i18n/tasks/base_task.rb +2 -0
  16. data/lib/i18n/tasks/command/commands/check_prism.rb +67 -15
  17. data/lib/i18n/tasks/command/commands/prune.rb +57 -0
  18. data/lib/i18n/tasks/commands.rb +2 -0
  19. data/lib/i18n/tasks/data/adapter/json_adapter.rb +1 -1
  20. data/lib/i18n/tasks/data/file_formats.rb +4 -1
  21. data/lib/i18n/tasks/data/router/conservative_router.rb +13 -22
  22. data/lib/i18n/tasks/data/router/pattern_router.rb +26 -14
  23. data/lib/i18n/tasks/data/tree/node.rb +3 -3
  24. data/lib/i18n/tasks/data/tree/siblings.rb +11 -3
  25. data/lib/i18n/tasks/data/tree/traversal.rb +20 -14
  26. data/lib/i18n/tasks/interpolations.rb +61 -3
  27. data/lib/i18n/tasks/logging.rb +1 -2
  28. data/lib/i18n/tasks/missing_keys.rb +7 -38
  29. data/lib/i18n/tasks/plural_keys.rb +56 -0
  30. data/lib/i18n/tasks/prune.rb +18 -0
  31. data/lib/i18n/tasks/references.rb +9 -1
  32. data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +11 -2
  33. data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +16 -2
  34. data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +6 -3
  35. data/lib/i18n/tasks/scanners/file_scanner.rb +4 -1
  36. data/lib/i18n/tasks/scanners/files/file_finder.rb +29 -6
  37. data/lib/i18n/tasks/scanners/files/parse_cache.rb +24 -0
  38. data/lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb +33 -0
  39. data/lib/i18n/tasks/scanners/prism_scanners/nodes.rb +57 -23
  40. data/lib/i18n/tasks/scanners/prism_scanners/visitor.rb +70 -16
  41. data/lib/i18n/tasks/scanners/relative_keys.rb +21 -11
  42. data/lib/i18n/tasks/scanners/ruby_scanner.rb +37 -5
  43. data/lib/i18n/tasks/split_key.rb +16 -6
  44. data/lib/i18n/tasks/translators/base_translator.rb +25 -23
  45. data/lib/i18n/tasks/translators/deepl_translator.rb +18 -15
  46. data/lib/i18n/tasks/translators/google_translator.rb +41 -14
  47. data/lib/i18n/tasks/translators/openai_translator.rb +31 -17
  48. data/lib/i18n/tasks/translators/watsonx_translator.rb +8 -10
  49. data/lib/i18n/tasks/translators/yandex_translator.rb +4 -0
  50. data/lib/i18n/tasks/unused_keys.rb +35 -5
  51. data/lib/i18n/tasks/used_keys.rb +6 -1
  52. data/lib/i18n/tasks/version.rb +1 -1
  53. data/script/report_test_results.rb +58 -0
  54. data/templates/config/i18n-tasks.yml +11 -3
  55. metadata +17 -7
@@ -90,15 +90,14 @@ module I18n::Tasks
90
90
 
91
91
  # Select the nodes for which the block returns true. Pre-order traversal.
92
92
  # @return [Siblings] a new tree
93
- def select_nodes(&block)
94
- tree = Siblings.new
93
+ def select_nodes(parent: nil, &block)
94
+ tree = Siblings.new(parent: parent)
95
95
  each do |node|
96
96
  next unless block.yield(node)
97
97
 
98
- tree.append! node.derive(
99
- parent: tree.parent,
100
- children: (node.children.select_nodes(&block).to_a if node.children)
101
- )
98
+ copy = node.derive(parent: parent, children: nil)
99
+ copy.children = node.children.select_nodes(parent: copy, &block) if node.children
100
+ tree.append! copy
102
101
  end
103
102
  tree
104
103
  end
@@ -167,34 +166,41 @@ module I18n::Tasks
167
166
  end
168
167
 
169
168
  def set_each_value!(val_pattern, key_pattern = nil, &value_proc) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
169
+ # A lone placeholder keeps the value's type, so that an array stays an array
170
+ lone_placeholder = val_pattern&.match(/\A%\{(\w+)\}\z/)&.[](1)&.to_sym
171
+ interpolate = lambda do |values|
172
+ lone_value = values[lone_placeholder] if lone_placeholder
173
+ return lone_value.dup unless lone_value.nil?
174
+
175
+ StringInterpolation.interpolate_soft(val_pattern, values)
176
+ end
170
177
  value_proc ||= proc do |node|
171
178
  node_value = node.value
172
179
  next node_value if node.reference?
173
180
 
181
+ kept_value = node_value.is_a?(String) ? node_value.presence : node_value
174
182
  human_key = ActiveSupport::Inflector.humanize(node.key.to_s)
175
183
  full_key = node.full_key
176
184
  default = (node.data[:occurrences] || []).detect { |o| o.default_arg.presence }.try(:default_arg)
177
185
  if default.is_a?(Hash)
178
186
  default.each_with_object({}) do |(k, v), h|
179
- h[k] = StringInterpolation.interpolate_soft(
180
- val_pattern,
187
+ h[k] = interpolate.call(
181
188
  value: node_value,
182
189
  human_key: human_key,
183
190
  key: full_key,
184
191
  default: v,
185
- value_or_human_key: node_value.presence || human_key,
186
- value_or_default_or_human_key: node_value.presence || v || human_key
192
+ value_or_human_key: kept_value.nil? ? human_key : kept_value,
193
+ value_or_default_or_human_key: kept_value.nil? ? v || human_key : kept_value
187
194
  )
188
195
  end
189
196
  else
190
- StringInterpolation.interpolate_soft(
191
- val_pattern,
197
+ interpolate.call(
192
198
  value: node_value,
193
199
  human_key: human_key,
194
200
  key: full_key,
195
201
  default: default,
196
- value_or_human_key: node_value.presence || human_key,
197
- value_or_default_or_human_key: node_value.presence || default || human_key
202
+ value_or_human_key: kept_value.nil? ? human_key : kept_value,
203
+ value_or_default_or_human_key: kept_value.nil? ? default || human_key : kept_value
198
204
  )
199
205
  end
200
206
  end
@@ -11,17 +11,34 @@ module I18n::Tasks
11
11
  locales ||= self.locales
12
12
  base_locale ||= self.base_locale
13
13
  result = empty_forest
14
- variable_regex = I18n::Tasks::Interpolations.variable_regex
14
+ plural_keys = Set.new
15
+ plural_nodes(data[base_locale]) do |base_node|
16
+ next unless base_node.parent?
17
+
18
+ group_key = base_node.full_key(root: false)
19
+ base_node.children.each { |form| plural_keys << "#{group_key}.#{form.key}" }
20
+ next if ignore_key?(group_key, :inconsistent_interpolations)
21
+
22
+ (locales - [base_locale]).each do |current_locale|
23
+ node = data[current_locale].first.children[group_key]
24
+ inconsistent_plural_forms(current_locale, base_node, node).each do |form|
25
+ next if ignore_key?("#{group_key}.#{form.key}", :inconsistent_interpolations)
26
+
27
+ result.merge!(form.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
28
+ end
29
+ end
30
+ end
15
31
 
16
32
  data[base_locale].key_values.each do |key, value|
33
+ next if plural_keys.include?(key)
17
34
  next if !value.is_a?(String) || ignore_key?(key, :inconsistent_interpolations)
18
35
 
19
- base_vars = Set.new(value.scan(variable_regex))
36
+ base_vars = interpolation_variables(value)
20
37
  (locales - [base_locale]).each do |current_locale|
21
38
  node = data[current_locale].first.children[key]
22
39
  next unless node&.value.is_a?(String)
23
40
 
24
- if base_vars != Set.new(node.value.scan(variable_regex))
41
+ if base_vars != interpolation_variables(node.value)
25
42
  result.merge!(node.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
26
43
  end
27
44
  end
@@ -31,6 +48,47 @@ module I18n::Tasks
31
48
  result
32
49
  end
33
50
 
51
+ COUNT_VARIABLE = "%{count}"
52
+ # The forms that are used for at most one count when the locale has no plural rule in rails-i18n.
53
+ SINGLE_COUNT_PLURAL_FORMS = %w[zero one two 0 1].freeze
54
+
55
+ # Plural forms are compared as a group, because a locale has different forms than the base locale, and its forms
56
+ # cover different counts: the Russian `one` is used for 1, 21, 31, ..., so it needs `%{count}` even when the
57
+ # English `one` spells the number out. A form may only use the variables of the base forms, and must keep the
58
+ # variables of the same base form (or of `other` if the base locale does not have it). It may drop `%{count}`
59
+ # if it is used for at most one count, and must have it if it is used for several. The counts each form is used
60
+ # for come from the locale's rule in rails-i18n.
61
+ #
62
+ # Keys that are not plural forms in the translated group have no base form to compare with, and are skipped.
63
+ #
64
+ # @return [Array<I18n::Tasks::Data::Tree::Node>] the inconsistent forms of `node`
65
+ def inconsistent_plural_forms(locale, base_node, node)
66
+ return [] unless node&.children?
67
+
68
+ base_forms = base_node.children.to_h { |form| [form.key, interpolation_variables(form.value)] }
69
+ base_vars = base_forms.values.reduce(Set.new, :|)
70
+ forms = node.children.select { |form| form.leaf? && plural_suffix?(form.key) && form.value.is_a?(String) }
71
+ form_counts = plural_form_counts(locale, node.children.map(&:key))
72
+ forms.select do |form|
73
+ vars = interpolation_variables(form.value)
74
+ required = (base_forms[form.key] || base_forms["other"] || base_vars) - [COUNT_VARIABLE]
75
+ if several_plural_counts?(form_counts, form.key) && base_vars.include?(COUNT_VARIABLE)
76
+ required << COUNT_VARIABLE
77
+ end
78
+ !vars.subset?(base_vars) || !required.subset?(vars)
79
+ end
80
+ end
81
+
82
+ def several_plural_counts?(form_counts, form_key)
83
+ return !SINGLE_COUNT_PLURAL_FORMS.include?(form_key) if form_counts.nil?
84
+
85
+ form_counts.fetch(form_key, 0) > 1
86
+ end
87
+
88
+ def interpolation_variables(value)
89
+ value.is_a?(String) ? Set.new(value.scan(I18n::Tasks::Interpolations.variable_regex)) : Set.new
90
+ end
91
+
34
92
  def reserved_interpolations(locales: nil)
35
93
  locales ||= self.locales
36
94
  result = empty_forest
@@ -25,8 +25,7 @@ module I18n::Tasks::Logging
25
25
  def log_stderr(*args)
26
26
  # We don't want output from different threads to get intermixed.
27
27
  MUTEX.synchronize do
28
- # Use $stderr directly to avoid issues with JRuby and thread safety
29
- $stderr.puts(*args) # rubocop:disable Style/StderrPuts
28
+ warn(*args)
30
29
  end
31
30
  end
32
31
 
@@ -51,8 +51,9 @@ module I18n::Tasks
51
51
  end
52
52
 
53
53
  def missing_used_forest(locales, _base = base_locale)
54
+ used = used_tree(strict: true)
54
55
  locales.inject(empty_forest) do |forest, locale|
55
- forest.merge! missing_used_tree(locale)
56
+ forest.merge! missing_used_tree(locale, used: used)
56
57
  end
57
58
  end
58
59
 
@@ -83,13 +84,7 @@ module I18n::Tasks
83
84
  @plural_keys_for_locale ||= {}
84
85
  return @plural_keys_for_locale[locale] if @plural_keys_for_locale.key?(locale)
85
86
 
86
- @plural_keys_for_locale[locale] = plural_keys_for_locale(locale)
87
- end
88
-
89
- # Loads rails-i18n pluralization config for the given locale.
90
- def load_rails_i18n_pluralization!(locale)
91
- path = File.join(Gem::Specification.find_by_name("rails-i18n").gem_dir, "rails", "pluralization", "#{locale}.rb")
92
- eval(File.read(path), binding, path) # rubocop:disable Security/Eval
87
+ @plural_keys_for_locale[locale] = Set.new(rails_i18n_plural_config(locale)&.dig(:keys) || [])
93
88
  end
94
89
 
95
90
  # keys present in compared_to, but not in locale
@@ -107,8 +102,9 @@ module I18n::Tasks
107
102
  end
108
103
 
109
104
  # keys used in the code missing translations in locale
110
- def missing_used_tree(locale)
111
- used_tree(strict: true).select_keys do |key, node|
105
+ def missing_used_tree(locale, used: nil)
106
+ used ||= used_tree(strict: true)
107
+ used.select_keys do |key, node|
112
108
  occurrences = node.data[:occurrences] || []
113
109
 
114
110
  # An occurrence may carry candidate keys (for relative lookups). If any
@@ -138,7 +134,7 @@ module I18n::Tasks
138
134
  end
139
135
 
140
136
  def locale_key_missing?(locale, key)
141
- !key_value?(key, locale) && !external_key?(key, locale) && !ignore_key?(key, :missing)
137
+ !key_value?(key, locale) && !external_key?(key, locale) && !ignore_key?(key, :missing, locale)
142
138
  end
143
139
 
144
140
  # @param [::I18n::Tasks::Data::Tree::Siblings] forest
@@ -170,32 +166,5 @@ module I18n::Tasks
170
166
  end
171
167
  forest
172
168
  end
173
-
174
- private
175
-
176
- def plural_keys_for_locale(locale)
177
- configuration = load_rails_i18n_pluralization!(locale)
178
- if configuration[locale.to_sym].nil?
179
- alternate_locale = alternate_locale_from(locale)
180
- return Set.new if configuration[alternate_locale.to_sym].nil?
181
-
182
- return set_from_rails_i18n_pluralization(configuration, alternate_locale)
183
- end
184
- set_from_rails_i18n_pluralization(configuration, locale)
185
- rescue SystemCallError, IOError
186
- Set.new
187
- end
188
-
189
- def alternate_locale_from(locale)
190
- re = /(\w{2})-*(\w{2,3})*/
191
- match = locale.match(re)
192
- language_code = match[1]
193
- country_code = match[2]
194
- "#{language_code}-#{country_code.upcase}"
195
- end
196
-
197
- def set_from_rails_i18n_pluralization(configuration, locale)
198
- Set.new(configuration[locale.to_sym][:i18n][:plural][:keys])
199
- end
200
169
  end
201
170
  end
@@ -60,4 +60,60 @@ module I18n::Tasks::PluralKeys
60
60
  def plural_suffix?(key)
61
61
  PLURAL_KEY_SUFFIXES.include?(key) || EXPLICIT_0_1.include?(key)
62
62
  end
63
+
64
+ # @param [String] locale
65
+ # @return [Hash, nil] the rails-i18n plural configuration of the locale (`:keys` and `:rule`), nil if it has none.
66
+ def rails_i18n_plural_config(locale)
67
+ @rails_i18n_plural_configs ||= {}
68
+ return @rails_i18n_plural_configs[locale] if @rails_i18n_plural_configs.key?(locale)
69
+
70
+ @rails_i18n_plural_configs[locale] = begin
71
+ configuration = load_rails_i18n_pluralization!(locale)
72
+ configuration = configuration[locale.to_sym] || configuration[alternate_locale_from(locale).to_sym]
73
+ configuration&.dig(:i18n, :plural)
74
+ rescue SystemCallError, IOError
75
+ nil
76
+ end
77
+ end
78
+
79
+ # Loads rails-i18n pluralization config for the given locale.
80
+ def load_rails_i18n_pluralization!(locale)
81
+ path = File.join(Gem::Specification.find_by_name("rails-i18n").gem_dir, "rails", "pluralization", "#{locale}.rb")
82
+ eval(File.read(path), binding, path) # rubocop:disable Security/Eval
83
+ end
84
+
85
+ # Counts from 0 up to this cover every plural rule in rails-i18n, as they look at `n % 100` at most.
86
+ PLURAL_RULE_COUNTS = (0..1000)
87
+
88
+ # @param [String] locale
89
+ # @param [Array<String>] form_keys the plural forms of a group in the locale
90
+ # @return [Hash{String => Integer}, nil] how many counts in PLURAL_RULE_COUNTS select each form,
91
+ # with the locale's rails-i18n rule, nil if the locale has no rule.
92
+ def plural_form_counts(locale, form_keys)
93
+ rule = rails_i18n_plural_config(locale)&.dig(:rule)
94
+ return unless rule.respond_to?(:call)
95
+
96
+ @plural_form_counts ||= {}
97
+ @plural_form_counts[[locale, form_keys.sort]] ||=
98
+ PLURAL_RULE_COUNTS.map { |count| plural_form_for_count(rule, form_keys, count) }.tally
99
+ end
100
+
101
+ private
102
+
103
+ # The form I18n::Backend::Pluralization picks for the count.
104
+ def plural_form_for_count(rule, form_keys, count)
105
+ return "zero" if count.zero? && form_keys.include?("zero")
106
+ return count.to_s if count <= 1 && form_keys.include?(count.to_s)
107
+
108
+ category = rule.call(count).to_s
109
+ form_keys.include?(category) ? category : "other"
110
+ end
111
+
112
+ def alternate_locale_from(locale)
113
+ re = /(\w{2})-*(\w{2,3})*/
114
+ match = locale.match(re)
115
+ language_code = match[1]
116
+ country_code = match[2]
117
+ "#{language_code}-#{country_code&.upcase}"
118
+ end
63
119
  end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18n::Tasks::Prune
4
+ def prunable_keys(base_locale:, locales:)
5
+ diff_forest = empty_forest
6
+ locales.each do |locale|
7
+ # Based on the docs, missing_diff_tree only returns keys that are in the second argument but not in the first
8
+ locale_forest =
9
+ missing_diff_tree(base_locale, locale).tap do |locale_forest|
10
+ locale_forest.mv_key!(compile_key_pattern(base_locale), locale, root: true)
11
+ end
12
+
13
+ diff_forest.merge!(locale_forest)
14
+ end
15
+
16
+ diff_forest
17
+ end
18
+ end
@@ -7,7 +7,7 @@ module I18n::Tasks
7
7
  # 2. Resolved references -- all the used references in their fully resolved form.
8
8
  # 3. Reference keys -- all the used reference keys.
9
9
  def process_references(usages,
10
- data_refs = merge_reference_trees(data_forest.select_keys { |_, node| node.reference? }))
10
+ data_refs = merge_reference_trees(reference_trees))
11
11
  fail ArgumentError, "usages must be a Data::Tree::Instance" unless usages.is_a?(Data::Tree::Siblings)
12
12
  fail ArgumentError, "all_references must be a Data::Tree::Instance" unless data_refs.is_a?(Data::Tree::Siblings)
13
13
 
@@ -30,6 +30,14 @@ module I18n::Tasks
30
30
 
31
31
  private
32
32
 
33
+ # Selecting the references of each locale before merging gives the same forest as
34
+ # data_forest.select_keys, without deep-copying every locale tree first.
35
+ def reference_trees
36
+ locales.inject(empty_forest) do |forest, locale|
37
+ forest.merge!(data[locale].select_keys { |_, node| node.reference? })
38
+ end
39
+ end
40
+
33
41
  # @param [I18n::Tasks::Data::Tree::Node] ref
34
42
  # @param [I18n::Tasks::Data::Tree::Node] usage
35
43
  # @param [I18n::Tasks::Data::Tree::Siblings] raw_refs
@@ -12,7 +12,8 @@ module I18n::Tasks::Scanners::AstMatchers
12
12
 
13
13
  protected
14
14
 
15
- # If the node type is of `%i(sym str int false true)`, return the value as a string.
15
+ # If the node type is of `%i(sym str int false true)`, or a `dstr` of string literals only
16
+ # (see `adjacent_string_literals?`), return the value as a string.
16
17
  # Otherwise, if `config[:strict]` is `false` and the type is of `%i(dstr dsym)`,
17
18
  # return the source as if it were a string.
18
19
  #
@@ -43,6 +44,8 @@ module I18n::Tasks::Scanners::AstMatchers
43
44
  # `nil` is returned when a dynamic value is encountered in strict mode. Propagate:
44
45
  return nil if str.nil?
45
46
  end
47
+ elsif adjacent_string_literals?(node)
48
+ node.children.map { |child| child.children[0] }.join
46
49
  elsif !@scanner.config[:strict] && %i[dsym dstr].include?(node.type)
47
50
  node.children.map do |child|
48
51
  if %i[sym str].include?(child.type)
@@ -54,6 +57,12 @@ module I18n::Tasks::Scanners::AstMatchers
54
57
  end
55
58
  end
56
59
 
60
+ # Adjacent string literals, e.g. `"a.b" \ ".c"`, and multi-line string literals are parsed
61
+ # as a `dstr` of `str` parts only: they are as static as a `str`.
62
+ def adjacent_string_literals?(node)
63
+ node.type == :dstr && node.children.any? && node.children.all? { |child| child.type == :str }
64
+ end
65
+
57
66
  # Extract the whole hash from a node of type `:hash`
58
67
  #
59
68
  # @param node [AST::Node] a node of type `:hash`.
@@ -93,7 +102,7 @@ module I18n::Tasks::Scanners::AstMatchers
93
102
  # @return [String, nil] `nil` is returned only when a dynamic value is encountered in strict mode.
94
103
  def extract_array_as_string(node, array_join_with:, array_flatten: false, array_reject_blank: false)
95
104
  children_strings = node.children.map do |child|
96
- if %i[sym str int true false].include?(child.type)
105
+ if %i[sym str int true false].include?(child.type) || adjacent_string_literals?(child)
97
106
  extract_string child
98
107
  else
99
108
  # ignore dynamic argument in strict mode
@@ -20,9 +20,23 @@ module I18n::Tasks::Scanners::AstMatchers
20
20
 
21
21
  value = children[2]
22
22
 
23
+ # Only process static string/symbol arguments; dynamic args (dstr/dsym) cannot be
24
+ # statically resolved and would produce a bogus key (e.g. "activerecord.attributes.product.").
25
+ return if value.nil?
26
+ return unless %i[str sym].include?(value.type) || adjacent_string_literals?(value)
27
+
23
28
  model_name = underscore(receiver.to_a.last)
24
29
  attribute = extract_string(value)
25
- key = "activerecord.attributes.#{model_name}.#{attribute}"
30
+ # Convert dots to slashes to match Rails' human_attribute_name behavior
31
+ # Rails converts "status.active" in Product.human_attribute_name("status.active")
32
+ # to look for activerecord.attributes.product/status.active
33
+ # The first dot becomes a slash: model_name.attribute -> model_name/first_part.rest
34
+ key = if attribute&.include?(".")
35
+ # Replace model_name.attribute with model_name/attribute for nested lookups
36
+ "activerecord.attributes.#{model_name}/#{attribute}"
37
+ else
38
+ "activerecord.attributes.#{model_name}.#{attribute}"
39
+ end
26
40
  [
27
41
  key,
28
42
  I18n::Tasks::Scanners::Results::Occurrence.from_range(
@@ -61,7 +75,7 @@ module I18n::Tasks::Scanners::AstMatchers
61
75
  end
62
76
 
63
77
  def underscore(value)
64
- value = value.dup.to_s
78
+ value = value.to_s.dup
65
79
  value.gsub!(/(.)([A-Z])/, '\1_\2')
66
80
  value.downcase!
67
81
  end
@@ -83,7 +83,7 @@ module I18n::Tasks::Scanners
83
83
  # Prism scanner, method called in RubyScanner
84
84
  def prism_parse_file(path)
85
85
  occurrences = []
86
- content = File.read(path)
86
+ content = read_file(path)
87
87
 
88
88
  content.scan(DEFAULT_REGEXP) do |indicator, code, _tailch, _rspace|
89
89
  match = Regexp.last_match
@@ -104,7 +104,7 @@ module I18n::Tasks::Scanners
104
104
  def process_code(path, code, content, start)
105
105
  return [] if code.strip.empty? # skip empty ERB tags
106
106
 
107
- process_prism_results(path, Prism.parse(code)).map do |key, occurrence|
107
+ process_prism_results(path, Prism.parse(code, **PRISM_PARSE_OPTIONS)).map do |key, occurrence|
108
108
  [
109
109
  key,
110
110
  occurrence_from_position(
@@ -121,7 +121,10 @@ module I18n::Tasks::Scanners
121
121
  def process_comments(path, code, content, start)
122
122
  return [] if code.strip.empty?
123
123
 
124
- parsed = Prism.parse(code.gsub("i18n-tasks-use ", "#i18n-tasks-use "))
124
+ # ERB comments are not executed, so only the `i18n-tasks-use` magic comments
125
+ # inside them are relevant. Turn every line into a Ruby comment so that
126
+ # commented-out code (e.g. `<%# t("key") %>`) is not reported as used.
127
+ parsed = Prism.parse(code.each_line.map { |line| "##{line.sub(/\A[ \t#]*/, "")}" }.join, **PRISM_PARSE_OPTIONS)
125
128
  process_prism_results(path, parsed).map do |key, occurrence|
126
129
  [
127
130
  key,
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "i18n/tasks/scanners/scanner"
4
+ require "i18n/tasks/scanners/files/parse_cache"
4
5
 
5
6
  module I18n::Tasks::Scanners
6
7
  # A base class for a scanner that analyses files.
@@ -13,11 +14,13 @@ module I18n::Tasks::Scanners
13
14
  def initialize(
14
15
  config: {},
15
16
  file_finder_provider: Files::CachingFileFinderProvider.new,
16
- file_reader: Files::CachingFileReader.new
17
+ file_reader: Files::CachingFileReader.new,
18
+ parse_cache: Files::ParseCache.new
17
19
  )
18
20
  super()
19
21
  @config = config
20
22
  @file_reader = file_reader
23
+ @parse_cache = parse_cache
21
24
  @file_finder = file_finder_provider.get(**config.slice(:paths, :only, :exclude))
22
25
  end
23
26
 
@@ -11,14 +11,14 @@ module I18n::Tasks::Scanners::Files
11
11
  # absolute or relative to the working directory.
12
12
  # @param only [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
13
13
  # Files not matching any of the inclusion patterns will be excluded.
14
- # @param exclude [Arry<String>] {File.fnmatch}-compatible patterns of files to exclude.
14
+ # @param exclude [Array<String>] {File.fnmatch}-compatible patterns of files to exclude.
15
15
  # Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
16
16
  def initialize(paths: ["."], only: nil, exclude: [])
17
17
  fail "paths argument is required" if paths.nil?
18
18
 
19
19
  @paths = paths
20
- @include = only
21
- @exclude = exclude || []
20
+ @include = only && split_extension_globs(only)
21
+ @exclude = split_extension_globs(exclude || [])
22
22
  end
23
23
 
24
24
  # Traverse the paths and yield the matching ones.
@@ -51,11 +51,34 @@ module I18n::Tasks::Scanners::Files
51
51
 
52
52
  private
53
53
 
54
+ # Paths found under `.` start with `./`, which neither `*.rb` nor `app/**` match: try them without it too.
54
55
  # @param path [String]
55
- # @param globs [Array<String>]
56
+ # @param split_globs [Array(Array<String>, Array<String>)] from {#split_extension_globs}
57
+ # @return [Boolean]
58
+ def path_fnmatch_any?(path, split_globs)
59
+ path_fnmatch_split_globs?(path, split_globs) ||
60
+ (path.start_with?("./") && path_fnmatch_split_globs?(path.delete_prefix("./"), split_globs))
61
+ end
62
+
63
+ # `File.fnmatch("*.png", path)` is `path.end_with?(".png")`, except for a path starting with a dot,
64
+ # which `*` does not match without `File::FNM_DOTMATCH`.
65
+ # @param path [String]
66
+ # @param split_globs [Array(Array<String>, Array<String>)] from {#split_extension_globs}
56
67
  # @return [Boolean]
57
- def path_fnmatch_any?(path, globs)
58
- globs.any? { |glob| File.fnmatch(glob, path) }
68
+ def path_fnmatch_split_globs?(path, split_globs)
69
+ suffixes, other_globs = split_globs
70
+ (!path.start_with?(".") && path.end_with?(*suffixes)) ||
71
+ other_globs.any? { |glob| File.fnmatch(glob, path) }
72
+ end
73
+
74
+ # ASCII only: `String#end_with?` raises on incompatible encodings, where `File.fnmatch` returns false.
75
+ EXTENSION_GLOB = /\A\*(\.[\w-]+)+\z/
76
+
77
+ # @param globs [Array<String>]
78
+ # @return [Array(Array<String>, Array<String>)] the suffixes of the extension globs, and the other globs
79
+ def split_extension_globs(globs)
80
+ extension_globs, other_globs = globs.partition { |glob| EXTENSION_GLOB.match?(glob) }
81
+ [extension_globs.map { |glob| glob.delete_prefix("*") }, other_globs]
59
82
  end
60
83
  end
61
84
  end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "i18n/tasks/concurrent/cache"
4
+
5
+ module I18n::Tasks::Scanners::Files
6
+ # Keeps the files parsed while {#retain} runs, so that scanning them again
7
+ # (strictly, then non-strictly) does not parse them twice.
8
+ #
9
+ # @note This class is thread-safe.
10
+ class ParseCache
11
+ def retain
12
+ @cache = ::I18n::Tasks::Concurrent::Cache.new
13
+ yield
14
+ ensure
15
+ @cache = nil
16
+ end
17
+
18
+ # @param key [Object] identifies the parse, e.g. the scanner class and the path.
19
+ # @yieldreturn the parse, computed only if not retained.
20
+ def fetch(key, &)
21
+ @cache ? @cache.fetch(key, &) : yield
22
+ end
23
+ end
24
+ end
@@ -16,11 +16,27 @@ module I18n::Tasks::Scanners::PrismScanners
16
16
  end
17
17
  end
18
18
 
19
+ def visit_local_variable_read_node(node)
20
+ # Keep this node to know that we cannot resolve this argument statically
21
+ node
22
+ end
23
+
24
+ def visit_constant_read_node(node)
25
+ # Keep this node to know that we cannot resolve this argument statically
26
+ node
27
+ end
28
+
19
29
  # Cannot handle arguments that are calls
20
30
  def visit_call_node(_node)
21
31
  nil
22
32
  end
23
33
 
34
+ # Shorthand hash syntax (e.g. `scope:` as shorthand for `scope: scope`) wraps
35
+ # the implicit value in an ImplicitNode — delegate to the inner node.
36
+ def visit_implicit_node(node)
37
+ visit(node.value)
38
+ end
39
+
24
40
  def visit_symbol_node(node)
25
41
  node.value
26
42
  end
@@ -29,6 +45,23 @@ module I18n::Tasks::Scanners::PrismScanners
29
45
  node.content
30
46
  end
31
47
 
48
+ # Interpolated nodes (e.g. `Product.human_attribute_name("status.#{status}")`) contain dynamic content that cannot
49
+ # be statically resolved: return nil. Adjacent string literals (e.g. `"a.b" \ ".c"`) are parsed as an
50
+ # interpolated node too, but are static.
51
+ def visit_interpolated_string_node(node)
52
+ return unless node.parts.all?(Prism::StringNode)
53
+
54
+ node.parts.map(&:content).join
55
+ end
56
+
57
+ def visit_interpolated_symbol_node(_node)
58
+ nil
59
+ end
60
+
61
+ def visit_interpolated_x_string_node(_node)
62
+ nil
63
+ end
64
+
32
65
  def visit_array_node(node)
33
66
  node.child_nodes.map { |child| visit(child) }
34
67
  end