i18n-tasks 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +321 -242
- data/Rakefile +5 -1
- data/benchmarks/bench_helper.rb +156 -0
- data/benchmarks/data_bench.rb +173 -0
- data/benchmarks/end_to_end_bench.rb +138 -0
- data/benchmarks/fixtures/generator.rb +257 -0
- data/benchmarks/results/baseline.json +250 -0
- data/benchmarks/run_all.rb +60 -0
- data/benchmarks/scanning_bench.rb +135 -0
- data/benchmarks/tree_bench.rb +205 -0
- data/config/locales/en.yml +10 -0
- data/config/locales/ru.yml +14 -0
- data/i18n-tasks.gemspec +1 -1
- data/lib/i18n/tasks/base_task.rb +2 -0
- data/lib/i18n/tasks/command/commands/check_prism.rb +67 -15
- data/lib/i18n/tasks/command/commands/prune.rb +57 -0
- data/lib/i18n/tasks/commands.rb +2 -0
- data/lib/i18n/tasks/data/adapter/json_adapter.rb +1 -1
- data/lib/i18n/tasks/data/file_formats.rb +4 -1
- data/lib/i18n/tasks/data/router/conservative_router.rb +13 -22
- data/lib/i18n/tasks/data/router/pattern_router.rb +26 -14
- data/lib/i18n/tasks/data/tree/node.rb +3 -3
- data/lib/i18n/tasks/data/tree/siblings.rb +11 -3
- data/lib/i18n/tasks/data/tree/traversal.rb +20 -14
- data/lib/i18n/tasks/interpolations.rb +61 -3
- data/lib/i18n/tasks/logging.rb +1 -2
- data/lib/i18n/tasks/missing_keys.rb +7 -38
- data/lib/i18n/tasks/plural_keys.rb +56 -0
- data/lib/i18n/tasks/prune.rb +18 -0
- data/lib/i18n/tasks/references.rb +9 -1
- data/lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb +11 -2
- data/lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb +16 -2
- data/lib/i18n/tasks/scanners/erb_ast_scanner.rb +6 -3
- data/lib/i18n/tasks/scanners/file_scanner.rb +4 -1
- data/lib/i18n/tasks/scanners/files/file_finder.rb +29 -6
- data/lib/i18n/tasks/scanners/files/parse_cache.rb +24 -0
- data/lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb +33 -0
- data/lib/i18n/tasks/scanners/prism_scanners/nodes.rb +57 -23
- data/lib/i18n/tasks/scanners/prism_scanners/visitor.rb +70 -16
- data/lib/i18n/tasks/scanners/relative_keys.rb +21 -11
- data/lib/i18n/tasks/scanners/ruby_scanner.rb +37 -5
- data/lib/i18n/tasks/split_key.rb +16 -6
- data/lib/i18n/tasks/translators/base_translator.rb +25 -23
- data/lib/i18n/tasks/translators/deepl_translator.rb +18 -15
- data/lib/i18n/tasks/translators/google_translator.rb +41 -14
- data/lib/i18n/tasks/translators/openai_translator.rb +31 -17
- data/lib/i18n/tasks/translators/watsonx_translator.rb +8 -10
- data/lib/i18n/tasks/translators/yandex_translator.rb +4 -0
- data/lib/i18n/tasks/unused_keys.rb +35 -5
- data/lib/i18n/tasks/used_keys.rb +6 -1
- data/lib/i18n/tasks/version.rb +1 -1
- data/script/report_test_results.rb +58 -0
- data/templates/config/i18n-tasks.yml +11 -3
- metadata +17 -7
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Micro-benchmarks for the tree data structure operations.
|
|
4
|
+
#
|
|
5
|
+
# These isolate pure tree performance with no I/O, making them useful for
|
|
6
|
+
# detecting regressions in the tree algorithms themselves.
|
|
7
|
+
#
|
|
8
|
+
# Usage:
|
|
9
|
+
# bundle exec ruby benchmarks/tree_bench.rb
|
|
10
|
+
# bundle exec ruby benchmarks/tree_bench.rb --save
|
|
11
|
+
# bundle exec ruby benchmarks/tree_bench.rb --compare
|
|
12
|
+
# bundle exec ruby benchmarks/tree_bench.rb --memory # include memory profiles
|
|
13
|
+
|
|
14
|
+
require_relative "bench_helper"
|
|
15
|
+
|
|
16
|
+
require "i18n/tasks/data/tree/siblings"
|
|
17
|
+
require "i18n/tasks/data/tree/node"
|
|
18
|
+
|
|
19
|
+
SAVE_RESULTS = ARGV.include?("--save")
|
|
20
|
+
COMPARE_RESULTS = ARGV.include?("--compare")
|
|
21
|
+
MEMORY_PROFILE = ARGV.include?("--memory")
|
|
22
|
+
|
|
23
|
+
Siblings = I18n::Tasks::Data::Tree::Siblings
|
|
24
|
+
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
# Helpers to build in-memory trees of various sizes without touching disk
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
def flat_keys(count)
|
|
30
|
+
sections = %w[users posts comments orders products admin auth errors shared mailers]
|
|
31
|
+
Array.new(count) do |i|
|
|
32
|
+
section = sections[i % sections.size]
|
|
33
|
+
"#{section}.group_#{i / sections.size}.item_#{i}.label"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def build_siblings(key_count)
|
|
38
|
+
Siblings.from_key_names(flat_keys(key_count))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def build_nested_hash(key_count, locale = "en")
|
|
42
|
+
keys = flat_keys(key_count)
|
|
43
|
+
hash = {locale => {}}
|
|
44
|
+
keys.each_with_index do |key, i|
|
|
45
|
+
parts = key.split(".")
|
|
46
|
+
node = hash[locale]
|
|
47
|
+
parts[0..-2].each do |p|
|
|
48
|
+
node[p] ||= {}
|
|
49
|
+
node = node[p]
|
|
50
|
+
end
|
|
51
|
+
node[parts.last] = "value_#{i}"
|
|
52
|
+
end
|
|
53
|
+
hash
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Pre-build fixtures so allocation is not counted
|
|
57
|
+
SMALL_KEYS = flat_keys(200).freeze
|
|
58
|
+
MEDIUM_KEYS = flat_keys(2_000).freeze
|
|
59
|
+
LARGE_KEYS = flat_keys(8_000).freeze
|
|
60
|
+
|
|
61
|
+
SMALL_HASH = build_nested_hash(200).freeze
|
|
62
|
+
MEDIUM_HASH = build_nested_hash(2_000).freeze
|
|
63
|
+
|
|
64
|
+
SMALL_TREE = build_siblings(200).freeze
|
|
65
|
+
MEDIUM_TREE = build_siblings(2_000).freeze
|
|
66
|
+
LARGE_TREE = build_siblings(8_000).freeze
|
|
67
|
+
|
|
68
|
+
SMALL_TREE_B = build_siblings(200).freeze
|
|
69
|
+
MEDIUM_TREE_B = build_siblings(2_000).freeze
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Construction benchmarks
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
BenchHelper.header("Tree construction")
|
|
76
|
+
|
|
77
|
+
construction_suite = Benchmark.ips do |x|
|
|
78
|
+
x.config(warmup: 3, time: 10)
|
|
79
|
+
|
|
80
|
+
x.report("from_key_names (200 keys)") do
|
|
81
|
+
Siblings.from_key_names(SMALL_KEYS)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
x.report("from_key_names (2k keys)") do
|
|
85
|
+
Siblings.from_key_names(MEDIUM_KEYS)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
x.report("from_key_names (8k keys)") do
|
|
89
|
+
Siblings.from_key_names(LARGE_KEYS)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
x.report("from_nested_hash (200 keys)") do
|
|
93
|
+
Siblings.from_nested_hash(SMALL_HASH)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
x.report("from_nested_hash (2k keys)") do
|
|
97
|
+
Siblings.from_nested_hash(MEDIUM_HASH)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
x.compare!
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# Merge benchmarks
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
BenchHelper.header("Tree merge")
|
|
108
|
+
|
|
109
|
+
merge_suite = Benchmark.ips do |x|
|
|
110
|
+
x.config(warmup: 3, time: 10)
|
|
111
|
+
|
|
112
|
+
x.report("merge! (200 into 200 keys)") do
|
|
113
|
+
SMALL_TREE.derive.merge!(SMALL_TREE_B)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
x.report("merge! (2k into 2k keys)") do
|
|
117
|
+
MEDIUM_TREE.derive.merge!(MEDIUM_TREE_B)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
x.compare!
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
# Traversal benchmarks
|
|
125
|
+
# ---------------------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
BenchHelper.header("Tree traversal")
|
|
128
|
+
|
|
129
|
+
traversal_suite = Benchmark.ips do |x|
|
|
130
|
+
x.config(warmup: 3, time: 10)
|
|
131
|
+
|
|
132
|
+
x.report("leaves (2k keys)") do
|
|
133
|
+
MEDIUM_TREE.leaves.to_a
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
x.report("key_names (2k keys)") do
|
|
137
|
+
MEDIUM_TREE.key_names
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
x.report("key_names (8k keys)") do
|
|
141
|
+
LARGE_TREE.key_names
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
x.report("select_keys (2k, match half)") do
|
|
145
|
+
i = 0
|
|
146
|
+
MEDIUM_TREE.select_keys { |_key, _node| (i += 1).odd? }
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
x.report("nodes block iteration (2k keys)") do
|
|
150
|
+
MEDIUM_TREE.nodes.to_a
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
x.compare!
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
# subtract_by_key! benchmark
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
BenchHelper.header("Tree subtract")
|
|
161
|
+
|
|
162
|
+
subtract_suite = Benchmark.ips do |x|
|
|
163
|
+
x.config(warmup: 3, time: 10)
|
|
164
|
+
|
|
165
|
+
x.report("subtract_by_key! (200 keys)") do
|
|
166
|
+
SMALL_TREE.derive.subtract_by_key!(SMALL_TREE_B)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
x.report("subtract_by_key! (2k keys)") do
|
|
170
|
+
MEDIUM_TREE.derive.subtract_by_key!(MEDIUM_TREE_B)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
x.compare!
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# ---------------------------------------------------------------------------
|
|
177
|
+
# Memory profiles (opt-in via --memory)
|
|
178
|
+
# ---------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
if MEMORY_PROFILE
|
|
181
|
+
BenchHelper.header("Memory — from_key_names (2k keys)")
|
|
182
|
+
MemoryProfiler.report { Siblings.from_key_names(MEDIUM_KEYS) }
|
|
183
|
+
.pretty_print(scale_bytes: true, detailed_report: false)
|
|
184
|
+
|
|
185
|
+
BenchHelper.header("Memory — from_nested_hash (2k keys)")
|
|
186
|
+
MemoryProfiler.report { Siblings.from_nested_hash(MEDIUM_HASH) }
|
|
187
|
+
.pretty_print(scale_bytes: true, detailed_report: false)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
passed = true
|
|
191
|
+
if SAVE_RESULTS
|
|
192
|
+
BenchHelper.save_results(construction_suite, "tree/construction")
|
|
193
|
+
BenchHelper.save_results(merge_suite, "tree/merge")
|
|
194
|
+
BenchHelper.save_results(traversal_suite, "tree/traversal")
|
|
195
|
+
BenchHelper.save_results(subtract_suite, "tree/subtract")
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
if COMPARE_RESULTS
|
|
199
|
+
passed = false unless BenchHelper.compare_baseline(construction_suite, "tree/construction")
|
|
200
|
+
passed = false unless BenchHelper.compare_baseline(merge_suite, "tree/merge")
|
|
201
|
+
passed = false unless BenchHelper.compare_baseline(traversal_suite, "tree/traversal")
|
|
202
|
+
passed = false unless BenchHelper.compare_baseline(subtract_suite, "tree/subtract")
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
exit(1) if COMPARE_RESULTS && !passed
|
data/config/locales/en.yml
CHANGED
|
@@ -51,6 +51,7 @@ en:
|
|
|
51
51
|
missing: show missing translations, optionally match a pattern
|
|
52
52
|
mv: rename/merge the keys in locale data that match the given pattern
|
|
53
53
|
normalize: 'normalize translation data: sort and move to the right files'
|
|
54
|
+
prune: removes keys from non-base locales that are not present in base locale
|
|
54
55
|
remove_unused: remove unused keys
|
|
55
56
|
rm: remove the keys in locale data that match the given pattern
|
|
56
57
|
translate_missing: translate missing keys with Google Translate or DeepL Pro, optionally match
|
|
@@ -121,6 +122,15 @@ en:
|
|
|
121
122
|
Set OpenAI API key via OPENAI_API_KEY environment variable or translation.openai_api_key
|
|
122
123
|
in config/i18n-tasks.yml. Get the key at https://openai.com/.
|
|
123
124
|
no_results: OpenAI returned no results.
|
|
125
|
+
prune:
|
|
126
|
+
count:
|
|
127
|
+
one: "%{count} key to prune in %{locales}."
|
|
128
|
+
other: "%{count} keys to prune in %{locales}."
|
|
129
|
+
zero: No keys to prune.
|
|
130
|
+
pruned:
|
|
131
|
+
one: "%{count} key pruned from %{locales}."
|
|
132
|
+
other: "%{count} keys pruned from %{locales}."
|
|
133
|
+
title: Pruning keys in %{locales} that are not present in %{base_locale}
|
|
124
134
|
remove_unused:
|
|
125
135
|
confirm:
|
|
126
136
|
one: "%{count} translation will be removed from %{locales}."
|
data/config/locales/ru.yml
CHANGED
|
@@ -50,6 +50,7 @@ ru:
|
|
|
50
50
|
missing: показать недостающие переводы
|
|
51
51
|
mv: переименовать / объединить ключи, которые соответствуют заданному шаблону
|
|
52
52
|
normalize: нормализовать файлы переводов (сортировка и распределение)
|
|
53
|
+
prune: удаляет ключи из небазовых локалей, которые отсутствуют в базовой локали
|
|
53
54
|
remove_unused: удалить неиспользуемые ключи
|
|
54
55
|
rm: удалить ключи, которые соответствуют заданному шаблону
|
|
55
56
|
translate_missing: перевести недостающие переводы с Google Translate / DeepL Pro
|
|
@@ -121,6 +122,19 @@ ru:
|
|
|
121
122
|
Установить ключ API Яндекса с помощью переменной среды OPENAI_API_KEY или translation.openai_api_key
|
|
122
123
|
в config / i18n-tasks.yml. Получите ключ по адресу https://openai.com/.
|
|
123
124
|
no_results: Яндекс не дал результатов.
|
|
125
|
+
prune:
|
|
126
|
+
count:
|
|
127
|
+
few: "%{count} ключа для удаления в %{locales}."
|
|
128
|
+
many: "%{count} ключей для удаления в %{locales}."
|
|
129
|
+
one: "%{count} ключ удаления в %{locales}."
|
|
130
|
+
other: "%{count} ключей для удаления в %{locales}."
|
|
131
|
+
zero: Нет ключей для удаления.
|
|
132
|
+
pruned:
|
|
133
|
+
few: "%{count} ключа удалено из %{locales}."
|
|
134
|
+
many: "%{count} ключeй удалены из %{locales}."
|
|
135
|
+
one: "%{count} ключ удалён из %{locales}."
|
|
136
|
+
other: "%{count} ключeй удалены из %{locales}."
|
|
137
|
+
title: Обрезка ключей в %{locales}, которые не присутствуют в %{base_locale}
|
|
124
138
|
remove_unused:
|
|
125
139
|
confirm:
|
|
126
140
|
few: Переводы (%{count}) будут удалены из %{locales}.
|
data/i18n-tasks.gemspec
CHANGED
|
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
|
|
|
45
45
|
s.add_dependency "highline", ">= 3.0.0"
|
|
46
46
|
s.add_dependency "i18n"
|
|
47
47
|
s.add_dependency "parser", ">= 3.2.2.1"
|
|
48
|
-
s.add_dependency "prism"
|
|
48
|
+
s.add_dependency "prism", ">= 0.28"
|
|
49
49
|
s.add_dependency "rails-i18n"
|
|
50
50
|
s.add_dependency "rainbow", ">= 2.2.2", "< 4.0"
|
|
51
51
|
s.add_dependency "ruby-progressbar", "~> 1.8", ">= 1.8.1"
|
data/lib/i18n/tasks/base_task.rb
CHANGED
|
@@ -19,6 +19,7 @@ require "i18n/tasks/string_interpolation"
|
|
|
19
19
|
require "i18n/tasks/data"
|
|
20
20
|
require "i18n/tasks/configuration"
|
|
21
21
|
require "i18n/tasks/stats"
|
|
22
|
+
require "i18n/tasks/prune"
|
|
22
23
|
|
|
23
24
|
module I18n
|
|
24
25
|
module Tasks
|
|
@@ -38,6 +39,7 @@ module I18n
|
|
|
38
39
|
include Configuration
|
|
39
40
|
include Data
|
|
40
41
|
include Stats
|
|
42
|
+
include Prune
|
|
41
43
|
|
|
42
44
|
def initialize(config_file: nil, **config)
|
|
43
45
|
@config_override = config_file
|
|
@@ -42,10 +42,18 @@ module I18n::Tasks
|
|
|
42
42
|
|
|
43
43
|
tree = i18n.used_in_source_tree
|
|
44
44
|
keys = tree.nodes.select { |n| n.data[:occurrences].present? }
|
|
45
|
-
results[label] =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
results[label] = {
|
|
46
|
+
keys: keys.each_with_object({}) do |node, h|
|
|
47
|
+
full_key = node.full_key(root: false)
|
|
48
|
+
h[full_key] = node.data[:occurrences]
|
|
49
|
+
end,
|
|
50
|
+
# Build a set of all keys including candidate keys
|
|
51
|
+
all_keys: keys.flat_map do |node|
|
|
52
|
+
full_key = node.full_key(root: false)
|
|
53
|
+
candidates = node.data[:occurrences].flat_map { |occ| occ.candidate_keys || [] }
|
|
54
|
+
[full_key] + candidates
|
|
55
|
+
end.uniq
|
|
56
|
+
}
|
|
49
57
|
end
|
|
50
58
|
|
|
51
59
|
# restore original config and caches
|
|
@@ -54,12 +62,29 @@ module I18n::Tasks
|
|
|
54
62
|
i18n.instance_variable_set(:@search_config, nil)
|
|
55
63
|
i18n.instance_variable_set(:@keys_used_in_source_tree, nil)
|
|
56
64
|
|
|
57
|
-
default_keys = results[:default].keys
|
|
58
|
-
prism_keys = results[:prism].keys
|
|
65
|
+
default_keys = results[:default][:keys].keys
|
|
66
|
+
prism_keys = results[:prism][:keys].keys
|
|
67
|
+
|
|
68
|
+
# Use all_keys (including candidate_keys) for comparison
|
|
69
|
+
default_all_keys = results[:default][:all_keys]
|
|
70
|
+
prism_all_keys = results[:prism][:all_keys]
|
|
71
|
+
|
|
72
|
+
# Calculate differences, but filter out keys that are candidate keys in the other parser
|
|
73
|
+
only_default_raw = (default_all_keys - prism_all_keys)
|
|
74
|
+
only_prism_raw = (prism_all_keys - default_all_keys)
|
|
75
|
+
|
|
76
|
+
# Filter out keys that exist as candidates in the other parser
|
|
77
|
+
only_default = only_default_raw.reject do |key|
|
|
78
|
+
# Check if this key is a candidate in prism results
|
|
79
|
+
results[:prism][:keys].any? { |_main_key, occs| occs.any? { |occ| occ.candidate_keys&.include?(key) } }
|
|
80
|
+
end.sort
|
|
59
81
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
only_prism = only_prism_raw.reject do |key|
|
|
83
|
+
# Check if this key is a candidate in default results
|
|
84
|
+
results[:default][:keys].any? { |_main_key, occs| occs.any? { |occ| occ.candidate_keys&.include?(key) } }
|
|
85
|
+
end.sort
|
|
86
|
+
|
|
87
|
+
both = (default_all_keys & prism_all_keys).sort
|
|
63
88
|
|
|
64
89
|
File.open(report_path, "w") do |f|
|
|
65
90
|
f.puts "# i18n-tasks check_prism report"
|
|
@@ -70,6 +95,8 @@ module I18n::Tasks
|
|
|
70
95
|
f.puts "Summary"
|
|
71
96
|
f.puts "- total keys (default parser): #{default_keys.size}"
|
|
72
97
|
f.puts "- total keys (prism): #{prism_keys.size}"
|
|
98
|
+
f.puts "- total keys including candidates (default parser): #{default_all_keys.size}"
|
|
99
|
+
f.puts "- total keys including candidates (prism): #{prism_all_keys.size}"
|
|
73
100
|
f.puts "- keys in both: #{both.size}"
|
|
74
101
|
f.puts "- keys only in default parser: #{only_default.size}"
|
|
75
102
|
f.puts "- keys only in prism: #{only_prism.size}"
|
|
@@ -79,11 +106,24 @@ module I18n::Tasks
|
|
|
79
106
|
f.puts "## Keys found by the default parser but NOT by Prism (#{only_default.size})"
|
|
80
107
|
only_default.each do |k|
|
|
81
108
|
f.puts "\n### #{k}"
|
|
82
|
-
|
|
109
|
+
# Find occurrences where this key appears (either as main key or candidate)
|
|
110
|
+
occs = results[:default][:keys][k] || []
|
|
111
|
+
if occs.empty?
|
|
112
|
+
# This key might be a candidate key, find which main key has it
|
|
113
|
+
results[:default][:keys].each do |main_key, main_occs|
|
|
114
|
+
main_occs.each do |occ|
|
|
115
|
+
if occ.candidate_keys&.include?(k)
|
|
116
|
+
occs << occ
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
occs.first(5).each do |occ|
|
|
83
122
|
src = (occ.raw_key || k).to_s
|
|
84
123
|
line = (occ.line || "").strip
|
|
85
124
|
highlighted = line.gsub(src) { |m| "`#{m}`" }
|
|
86
|
-
|
|
125
|
+
candidate_info = occ.candidate_keys ? " (candidates: #{occ.candidate_keys.join(", ")})" : ""
|
|
126
|
+
f.puts "- #{occ.path}:#{occ.line_num} `#{src}`#{candidate_info} — #{highlighted}"
|
|
87
127
|
end
|
|
88
128
|
end
|
|
89
129
|
end
|
|
@@ -92,21 +132,33 @@ module I18n::Tasks
|
|
|
92
132
|
f.puts "## Keys found by Prism but NOT by the default parser (#{only_prism.size})"
|
|
93
133
|
only_prism.each do |k|
|
|
94
134
|
f.puts "\n### #{k}"
|
|
95
|
-
|
|
135
|
+
# Find occurrences where this key appears (either as main key or candidate)
|
|
136
|
+
occs = results[:prism][:keys][k] || []
|
|
137
|
+
if occs.empty?
|
|
138
|
+
# This key might be a candidate key, find which main key has it
|
|
139
|
+
results[:prism][:keys].each do |main_key, main_occs|
|
|
140
|
+
main_occs.each do |occ|
|
|
141
|
+
if occ.candidate_keys&.include?(k)
|
|
142
|
+
occs << occ
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
occs.each do |occ|
|
|
96
148
|
src = (occ.raw_key || k).to_s
|
|
97
149
|
line = (occ.line || "").strip
|
|
98
150
|
highlighted = line.gsub(src) { |m| "`#{m}`" }
|
|
99
|
-
|
|
151
|
+
candidate_info = occ.candidate_keys ? " (candidates: #{occ.candidate_keys.join(", ")})" : ""
|
|
152
|
+
f.puts "- #{occ.path}:#{occ.line_num} `#{src}`#{candidate_info} — #{highlighted}"
|
|
100
153
|
end
|
|
101
154
|
end
|
|
102
155
|
end
|
|
103
156
|
|
|
104
157
|
f.puts "\n## Notes"
|
|
105
|
-
f.puts "- This report compares keys discovered by the project default parser and by Prism (
|
|
158
|
+
f.puts "- This report compares keys discovered by the project default parser and by Prism (#{chosen_prism_mode} mode)."
|
|
106
159
|
end
|
|
107
160
|
|
|
108
161
|
log_stderr "Wrote check_prism report: #{report_path}"
|
|
109
|
-
puts File.read(report_path)
|
|
110
162
|
end
|
|
111
163
|
|
|
112
164
|
private
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module I18n::Tasks
|
|
2
|
+
module Command
|
|
3
|
+
module Commands
|
|
4
|
+
module Prune
|
|
5
|
+
include ::I18n::Tasks::Command::Collection
|
|
6
|
+
|
|
7
|
+
arg :keep_order,
|
|
8
|
+
"-k",
|
|
9
|
+
"--keep-order",
|
|
10
|
+
t("i18n_tasks.cmd.args.desc.keep_order")
|
|
11
|
+
|
|
12
|
+
cmd :prune, desc: t("i18n_tasks.cmd.desc.prune"), args: %i[confirm keep_order]
|
|
13
|
+
def prune(opts = {})
|
|
14
|
+
diff_forest = i18n.prunable_keys(
|
|
15
|
+
base_locale: i18n.base_locale,
|
|
16
|
+
locales: i18n.locales - [base_locale]
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
print_info t("i18n_tasks.prune.title", locales: locales.join(", "), base_locale: base_locale)
|
|
20
|
+
|
|
21
|
+
if diff_forest.empty?
|
|
22
|
+
print_success(t("i18n_tasks.prune.count", count: 0))
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
count = diff_forest.root_key_values.size
|
|
27
|
+
|
|
28
|
+
terminal_report.show_tree(diff_forest)
|
|
29
|
+
print_error(t("i18n_tasks.prune.count", count:, locales: locales.join(", ")))
|
|
30
|
+
|
|
31
|
+
# The user should confirm before we proceed with the deletion unless the force option is set
|
|
32
|
+
unless opts[:confirm] || agree(t("i18n_tasks.common.continue_q"))
|
|
33
|
+
return
|
|
34
|
+
end
|
|
35
|
+
print_success(t("i18n_tasks.prune.pruned", count:, locales: locales.join(", ")))
|
|
36
|
+
|
|
37
|
+
i18n.data.config = i18n.data.config.merge(sort: false) if opts[:"keep-order"]
|
|
38
|
+
i18n.data.remove_by_key!(diff_forest)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def print_success(message)
|
|
44
|
+
log_stderr(Rainbow("✓ #{message}").green.bright)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def print_error(message)
|
|
48
|
+
log_stderr(Rainbow(message).red.bright)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def print_info(message)
|
|
52
|
+
log_stderr(message)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
data/lib/i18n/tasks/commands.rb
CHANGED
|
@@ -11,6 +11,7 @@ require "i18n/tasks/command/commands/data"
|
|
|
11
11
|
require "i18n/tasks/command/commands/tree"
|
|
12
12
|
require "i18n/tasks/command/commands/meta"
|
|
13
13
|
require "i18n/tasks/command/commands/check_prism"
|
|
14
|
+
require "i18n/tasks/command/commands/prune"
|
|
14
15
|
require "i18n/tasks/command/commander"
|
|
15
16
|
|
|
16
17
|
module I18n::Tasks
|
|
@@ -25,6 +26,7 @@ module I18n::Tasks
|
|
|
25
26
|
include Command::Commands::Tree
|
|
26
27
|
include Command::Commands::Meta
|
|
27
28
|
include Command::Commands::CheckPrism
|
|
29
|
+
include Command::Commands::Prune
|
|
28
30
|
|
|
29
31
|
require "highline/import"
|
|
30
32
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
|
+
require "active_support/core_ext/file/atomic"
|
|
4
5
|
|
|
5
6
|
module I18n
|
|
6
7
|
module Tasks
|
|
@@ -55,7 +56,9 @@ module I18n
|
|
|
55
56
|
return if File.file?(path) && content == read_file(path)
|
|
56
57
|
|
|
57
58
|
::FileUtils.mkpath(File.dirname(path))
|
|
58
|
-
::File.
|
|
59
|
+
::File.atomic_write(path) do |file|
|
|
60
|
+
file.write(content)
|
|
61
|
+
end
|
|
59
62
|
end
|
|
60
63
|
|
|
61
64
|
def normalized?(path, tree)
|
|
@@ -16,33 +16,23 @@ module I18n::Tasks
|
|
|
16
16
|
def route(locale, forest, &block) # rubocop:disable Metrics/AbcSize
|
|
17
17
|
return to_enum(:route, locale, forest) unless block
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
forest.keys do |key,
|
|
22
|
-
|
|
19
|
+
locale = locale.to_s
|
|
20
|
+
leaves_by_path = Hash.new { |hash, path| hash[path] = [] }
|
|
21
|
+
forest.keys do |key, node|
|
|
22
|
+
key_parts = node.walk_from_root.drop(1).map(&:key)
|
|
23
|
+
path = key_path(locale, key_parts)
|
|
23
24
|
# infer from another locale
|
|
24
25
|
unless path
|
|
25
|
-
inferred_from = (locales - [locale]).detect { |loc| path = key_path(loc,
|
|
26
|
+
inferred_from = (locales - [locale]).detect { |loc| path = key_path(loc, key_parts) }
|
|
26
27
|
path = LocalePathname.replace_locale(path, inferred_from, locale) if inferred_from
|
|
27
28
|
end
|
|
28
|
-
key_with_locale = "#{locale}.#{key}"
|
|
29
|
-
if path
|
|
30
|
-
out[path] << key_with_locale
|
|
31
|
-
else
|
|
32
|
-
not_found << key_with_locale
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
if not_found.present?
|
|
37
29
|
# fall back to pattern router
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
out[path] += tree.key_names(root: true)
|
|
41
|
-
end
|
|
30
|
+
path ||= pattern_path(locale, key)
|
|
31
|
+
leaves_by_path[path] << node
|
|
42
32
|
end
|
|
43
33
|
|
|
44
|
-
|
|
45
|
-
block.yield dest, forest
|
|
34
|
+
leaves_by_path.each do |dest, leaves|
|
|
35
|
+
block.yield dest, select_leaves(forest, leaves)
|
|
46
36
|
end
|
|
47
37
|
end
|
|
48
38
|
|
|
@@ -52,8 +42,9 @@ module I18n::Tasks
|
|
|
52
42
|
adapter[base_locale]
|
|
53
43
|
end
|
|
54
44
|
|
|
55
|
-
|
|
56
|
-
|
|
45
|
+
# Looked up by key parts: splitting the dotted key would lose blank keys (e.g. `''`) and split keys containing a dot.
|
|
46
|
+
def key_path(locale, key_parts)
|
|
47
|
+
adapter[locale].dig(locale, *key_parts).try(:data).try(:[], :path)
|
|
57
48
|
end
|
|
58
49
|
|
|
59
50
|
attr_reader :adapter, :base_locale, :locales
|
|
@@ -32,24 +32,36 @@ module I18n::Tasks
|
|
|
32
32
|
return to_enum(:route, locale, forest) unless block
|
|
33
33
|
|
|
34
34
|
locale = locale.to_s
|
|
35
|
-
|
|
36
|
-
forest.keys do |key,
|
|
37
|
-
|
|
38
|
-
if pattern
|
|
39
|
-
key_match = $~
|
|
40
|
-
path = format(path, locale: locale)
|
|
41
|
-
path.gsub!(/\\\d+/) { |m| key_match[m[1..].to_i] }
|
|
42
|
-
(out[path] ||= Set.new) << "#{locale}.#{key}"
|
|
43
|
-
else
|
|
44
|
-
fail CommandError, "Cannot route key #{key}. Routes are #{@routes_config.inspect}"
|
|
45
|
-
end
|
|
35
|
+
leaves_by_path = Hash.new { |hash, path| hash[path] = [] }
|
|
36
|
+
forest.keys do |key, node|
|
|
37
|
+
leaves_by_path[pattern_path(locale, key)] << node
|
|
46
38
|
end
|
|
47
|
-
|
|
48
|
-
block.yield dest,
|
|
49
|
-
forest.select_keys(root: true) { |key, _| keys.include?(key) }
|
|
39
|
+
leaves_by_path.each do |dest, leaves|
|
|
40
|
+
block.yield dest, select_leaves(forest, leaves)
|
|
50
41
|
end
|
|
51
42
|
end
|
|
52
43
|
|
|
44
|
+
protected
|
|
45
|
+
|
|
46
|
+
def pattern_path(locale, key)
|
|
47
|
+
pattern, path = routes.detect { |route| route[0] =~ key }
|
|
48
|
+
fail CommandError, "Cannot route key #{key}. Routes are #{@routes_config.inspect}" unless pattern
|
|
49
|
+
|
|
50
|
+
key_match = $~
|
|
51
|
+
path = format(path, locale: locale)
|
|
52
|
+
path.gsub!(/\\\d+/) { |m| key_match[m[1..].to_i] }
|
|
53
|
+
path
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @return [Siblings] the subtree of forest holding these leaves, without walking the rest of the forest
|
|
57
|
+
def select_leaves(forest, leaves)
|
|
58
|
+
selected = Set.new
|
|
59
|
+
leaves.each do |leaf|
|
|
60
|
+
leaf.walk_to_root { |node| break unless selected.add?(node) }
|
|
61
|
+
end
|
|
62
|
+
forest.select_nodes { |node| selected.include?(node) }
|
|
63
|
+
end
|
|
64
|
+
|
|
53
65
|
private
|
|
54
66
|
|
|
55
67
|
def compile_routes(routes)
|
|
@@ -192,10 +192,10 @@ module I18n::Tasks::Data::Tree
|
|
|
192
192
|
|
|
193
193
|
class << self
|
|
194
194
|
# value can be a nested hash
|
|
195
|
-
def from_key_value(key, value)
|
|
196
|
-
Node.new(key: key.try(:to_s)).tap do |node|
|
|
195
|
+
def from_key_value(key, value, parent: nil)
|
|
196
|
+
Node.new(key: key.try(:to_s), parent: parent).tap do |node|
|
|
197
197
|
if value.is_a?(Hash)
|
|
198
|
-
node.children = Siblings.from_nested_hash(value)
|
|
198
|
+
node.children = Siblings.from_nested_hash(value, parent: node)
|
|
199
199
|
else
|
|
200
200
|
node.value = value
|
|
201
201
|
end
|
|
@@ -81,9 +81,17 @@ module I18n::Tasks::Data::Tree
|
|
|
81
81
|
|
|
82
82
|
# @return [Node] by full key
|
|
83
83
|
def get(full_key)
|
|
84
|
-
|
|
84
|
+
dig(*split_key(full_key.to_s))
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# @return [Node] by key parts, which may be blank or contain a dot
|
|
88
|
+
def dig(first_key, *rest)
|
|
85
89
|
node = key_to_node[first_key]
|
|
86
|
-
|
|
90
|
+
rest.each do |key|
|
|
91
|
+
return nil unless node&.children
|
|
92
|
+
|
|
93
|
+
node = node.children.key_to_node[key]
|
|
94
|
+
end
|
|
87
95
|
node
|
|
88
96
|
end
|
|
89
97
|
|
|
@@ -301,7 +309,7 @@ module I18n::Tasks::Data::Tree
|
|
|
301
309
|
parse_parent_opt!(opts)
|
|
302
310
|
fail I18n::Tasks::CommandError, "invalid tree #{hash.inspect}" unless hash.respond_to?(:map)
|
|
303
311
|
|
|
304
|
-
opts[:nodes] = hash.map { |key, value| Node.from_key_value key, value }
|
|
312
|
+
opts[:nodes] = hash.map { |key, value| Node.from_key_value key, value, parent: opts[:parent] }
|
|
305
313
|
Siblings.new(opts)
|
|
306
314
|
end
|
|
307
315
|
|