i18n-context-generator 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +163 -26
- data/lib/i18n_context_generator/android_resource.rb +209 -0
- data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
- data/lib/i18n_context_generator/cache.rb +40 -8
- data/lib/i18n_context_generator/changed_location.rb +55 -0
- data/lib/i18n_context_generator/cli.rb +163 -76
- data/lib/i18n_context_generator/config/cli_values.rb +33 -0
- data/lib/i18n_context_generator/config/defaults.rb +35 -0
- data/lib/i18n_context_generator/config/schema.rb +225 -0
- data/lib/i18n_context_generator/config/serialization.rb +64 -0
- data/lib/i18n_context_generator/config/validation.rb +249 -0
- data/lib/i18n_context_generator/config.rb +293 -102
- data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
- data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
- data/lib/i18n_context_generator/context_extractor/run_logging.rb +45 -6
- data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
- data/lib/i18n_context_generator/context_extractor/source_filters.rb +51 -5
- data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
- data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
- data/lib/i18n_context_generator/context_extractor.rb +207 -216
- data/lib/i18n_context_generator/file_classifier.rb +72 -0
- data/lib/i18n_context_generator/generated_comment.rb +32 -0
- data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
- data/lib/i18n_context_generator/git_diff.rb +224 -95
- data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
- data/lib/i18n_context_generator/llm/client.rb +304 -92
- data/lib/i18n_context_generator/llm/openai.rb +92 -51
- data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
- data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
- data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
- data/lib/i18n_context_generator/localization_syntax.rb +246 -0
- data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
- data/lib/i18n_context_generator/parsers/base.rb +7 -4
- data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
- data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
- data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
- data/lib/i18n_context_generator/path_policy.rb +107 -0
- data/lib/i18n_context_generator/platform_validator.rb +24 -50
- data/lib/i18n_context_generator/run_metrics.rb +47 -0
- data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
- data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
- data/lib/i18n_context_generator/searcher/source_discovery.rb +109 -64
- data/lib/i18n_context_generator/searcher.rb +61 -205
- data/lib/i18n_context_generator/supplemental_context.rb +77 -0
- data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
- data/lib/i18n_context_generator/version.rb +1 -1
- data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
- data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
- data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
- data/lib/i18n_context_generator/writers/helpers.rb +7 -29
- data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
- data/lib/i18n_context_generator/writers/preview.rb +80 -0
- data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
- data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
- data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
- data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
- data/lib/i18n_context_generator/xcstrings_document.rb +248 -0
- data/lib/i18n_context_generator/xml_scanner.rb +38 -0
- data/lib/i18n_context_generator.rb +20 -4
- metadata +59 -12
|
@@ -4,26 +4,7 @@ module I18nContextGenerator
|
|
|
4
4
|
class Searcher
|
|
5
5
|
# Source-discovery helpers used for source-first extraction runs.
|
|
6
6
|
module SourceDiscovery
|
|
7
|
-
|
|
8
|
-
/NSLocalizedString\s*\(\s*@?["'](?<key>[^"']+)["'](?:.*?comment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["'])?/,
|
|
9
|
-
/String\s*\(\s*localized:\s*["'](?<key>[^"']+)["'](?:.*?comment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["'])?/,
|
|
10
|
-
/LocalizedStringKey\s*\(\s*["'](?<key>[^"']+)["']\s*\)/,
|
|
11
|
-
/:\s*LocalizedStringKey\s*=\s*["'](?<key>[^"']+)["']/,
|
|
12
|
-
/Text\s*\(\s*["'](?<key>[^"']+)["']/,
|
|
13
|
-
/["'](?<key>[^"']+)["']\.localized\b/
|
|
14
|
-
].freeze
|
|
15
|
-
|
|
16
|
-
IOS_MULTILINE_DISCOVERY_PATTERNS = [
|
|
17
|
-
/NSLocalizedString\s*\(\s*@?["'](?<key>[^"']+)["'](?:(?:(?!\)\s*[),]?)[\s\S])*?comment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["'])?/,
|
|
18
|
-
/String\s*\(\s*localized:\s*["'](?<key>[^"']+)["'](?:(?:(?!\)\s*[),]?)[\s\S])*?comment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["'])?/,
|
|
19
|
-
/Text\s*\(\s*LocalizedStringKey\s*\(\s*["'](?<key>[^"']+)["']\s*\)\s*\)/
|
|
20
|
-
].freeze
|
|
21
|
-
|
|
22
|
-
ANDROID_DISCOVERY_PATTERN = %r{
|
|
23
|
-
R\.string\.(\w+)\b|
|
|
24
|
-
@string/([\w.]+)\b|
|
|
25
|
-
[(\s,=]string\.(\w+)\b
|
|
26
|
-
}x
|
|
7
|
+
IOS_COMMENT_ARGUMENT_PATTERN = /\bcomment:\s*["'](?<comment>(?:\\.|[^"'\\])*)["']/
|
|
27
8
|
|
|
28
9
|
def discover_localization_entries
|
|
29
10
|
entries = discover_files.flat_map do |file|
|
|
@@ -55,31 +36,37 @@ module I18nContextGenerator
|
|
|
55
36
|
|
|
56
37
|
def deduplicate_discovered_entries(entries)
|
|
57
38
|
entries.each_with_object({}) do |entry, deduplicated_entries|
|
|
58
|
-
|
|
39
|
+
identity = [entry.resource_type, entry.key]
|
|
40
|
+
existing_entry = deduplicated_entries[identity]
|
|
59
41
|
if existing_entry.nil?
|
|
60
|
-
deduplicated_entries[
|
|
42
|
+
deduplicated_entries[identity] = entry
|
|
61
43
|
next
|
|
62
44
|
end
|
|
63
45
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
46
|
+
preferred_entry = if existing_entry.comment.to_s.empty? && !entry.comment.to_s.empty?
|
|
47
|
+
entry
|
|
48
|
+
else
|
|
49
|
+
existing_entry
|
|
50
|
+
end
|
|
51
|
+
deduplicated_entries[identity] = DiscoveredLocalization.new(
|
|
52
|
+
key: preferred_entry.key,
|
|
53
|
+
file: preferred_entry.file,
|
|
54
|
+
line: preferred_entry.line,
|
|
55
|
+
text: preferred_entry.text,
|
|
56
|
+
comment: preferred_entry.comment,
|
|
57
|
+
resource_type: preferred_entry.resource_type,
|
|
58
|
+
locations: (existing_entry.locations + entry.locations).uniq,
|
|
59
|
+
location_groups: (existing_entry.location_groups + entry.location_groups).uniq
|
|
60
|
+
)
|
|
67
61
|
end.values
|
|
68
62
|
end
|
|
69
63
|
|
|
70
64
|
def platform_for_file(file)
|
|
71
|
-
|
|
72
|
-
when '.swift', '.m', '.mm', '.h'
|
|
73
|
-
:ios
|
|
74
|
-
when '.kt', '.java'
|
|
75
|
-
:android
|
|
76
|
-
when '.xml'
|
|
77
|
-
file.include?('/res/') ? :android : nil
|
|
78
|
-
end
|
|
65
|
+
FileClassifier.searchable_platform(file, platform_hint: @platform)
|
|
79
66
|
end
|
|
80
67
|
|
|
81
68
|
def discover_ios_entries(file)
|
|
82
|
-
lines =
|
|
69
|
+
lines = searchable_file_lines(file)
|
|
83
70
|
entries = []
|
|
84
71
|
index = 0
|
|
85
72
|
|
|
@@ -94,43 +81,63 @@ module I18nContextGenerator
|
|
|
94
81
|
end
|
|
95
82
|
|
|
96
83
|
def extract_ios_entry(lines, file, index, line)
|
|
84
|
+
if @localization_syntax.ios_call_start_patterns.any? { |pattern| pattern.match?(line) }
|
|
85
|
+
next_index, discovered_entry = extract_ios_multiline_entry(lines, file, index)
|
|
86
|
+
return [next_index, discovered_entry] if discovered_entry
|
|
87
|
+
end
|
|
88
|
+
|
|
97
89
|
if (entry = extract_ios_single_line_entry(file, index, line))
|
|
98
90
|
return [index + 1, entry]
|
|
99
91
|
end
|
|
100
92
|
|
|
101
|
-
return extract_ios_multiline_entry(lines, file, index) if IOS_FUNCTION_OPENERS.any? { |opener| opener.match?(line) }
|
|
102
|
-
|
|
103
93
|
[index + 1, nil]
|
|
104
94
|
end
|
|
105
95
|
|
|
106
96
|
def extract_ios_single_line_entry(file, index, line)
|
|
107
|
-
pattern =
|
|
97
|
+
pattern = @localization_syntax.ios_single_line_discovery_patterns.find { |candidate| candidate.match?(line) }
|
|
108
98
|
return unless pattern
|
|
109
99
|
|
|
110
|
-
build_ios_discovered_entry(
|
|
100
|
+
build_ios_discovered_entry(
|
|
101
|
+
file,
|
|
102
|
+
index,
|
|
103
|
+
pattern.match(line),
|
|
104
|
+
comment_match: IOS_COMMENT_ARGUMENT_PATTERN.match(line)
|
|
105
|
+
)
|
|
111
106
|
end
|
|
112
107
|
|
|
113
108
|
def extract_ios_multiline_entry(lines, file, start_index, lookahead: 8)
|
|
114
|
-
end_index =
|
|
109
|
+
end_index = ios_call_end_index(lines, start_index, lookahead: lookahead)
|
|
115
110
|
snippet = lines[start_index..end_index].join("\n")
|
|
116
|
-
pattern =
|
|
111
|
+
pattern = @localization_syntax.ios_multiline_discovery_patterns.find { |candidate| candidate.match?(snippet) }
|
|
117
112
|
return [start_index + 1, nil] unless pattern
|
|
118
113
|
|
|
119
114
|
match = pattern.match(snippet)
|
|
120
115
|
key_line_index = locate_key_line(lines, start_index, end_index, match[:key])
|
|
116
|
+
comment_match = IOS_COMMENT_ARGUMENT_PATTERN.match(snippet)
|
|
117
|
+
locations = (start_index..end_index).map { |index| "#{file}:#{index + 1}" }
|
|
121
118
|
|
|
122
119
|
[
|
|
123
|
-
|
|
124
|
-
build_ios_discovered_entry(
|
|
120
|
+
end_index + 1,
|
|
121
|
+
build_ios_discovered_entry(
|
|
122
|
+
file,
|
|
123
|
+
key_line_index || start_index,
|
|
124
|
+
match,
|
|
125
|
+
comment_match: comment_match,
|
|
126
|
+
locations: locations
|
|
127
|
+
)
|
|
125
128
|
]
|
|
126
129
|
end
|
|
127
130
|
|
|
128
|
-
def build_ios_discovered_entry(file, index, match)
|
|
129
|
-
key = match[:key]
|
|
131
|
+
def build_ios_discovered_entry(file, index, match, comment_match: nil, locations: nil)
|
|
132
|
+
key = unescape_source_string(match[:key])
|
|
130
133
|
return if key.nil? || key.empty?
|
|
131
134
|
|
|
132
135
|
text = match.names.include?('text') ? match[:text] : nil
|
|
133
|
-
comment =
|
|
136
|
+
comment = if comment_match
|
|
137
|
+
unescape_source_string(comment_match[:comment])
|
|
138
|
+
elsif match.names.include?('comment')
|
|
139
|
+
unescape_source_string(match[:comment])
|
|
140
|
+
end
|
|
134
141
|
text = unescape_source_string(text) if text
|
|
135
142
|
|
|
136
143
|
DiscoveredLocalization.new(
|
|
@@ -138,10 +145,52 @@ module I18nContextGenerator
|
|
|
138
145
|
file: file,
|
|
139
146
|
line: index + 1,
|
|
140
147
|
text: text,
|
|
141
|
-
comment: comment
|
|
148
|
+
comment: comment,
|
|
149
|
+
locations: locations
|
|
142
150
|
)
|
|
143
151
|
end
|
|
144
152
|
|
|
153
|
+
def ios_call_end_index(lines, start_index, lookahead:)
|
|
154
|
+
maximum_index = [lines.length - 1, start_index + lookahead].min
|
|
155
|
+
depth = 0
|
|
156
|
+
found_opening = false
|
|
157
|
+
|
|
158
|
+
(start_index..maximum_index).each do |index|
|
|
159
|
+
parenthesis_delta(lines[index]).each do |delta|
|
|
160
|
+
found_opening = true if delta.positive?
|
|
161
|
+
depth += delta
|
|
162
|
+
end
|
|
163
|
+
return index if found_opening && depth <= 0
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
maximum_index
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def parenthesis_delta(line)
|
|
170
|
+
deltas = []
|
|
171
|
+
quote = nil
|
|
172
|
+
escaped = false
|
|
173
|
+
|
|
174
|
+
line.each_char do |character|
|
|
175
|
+
if quote
|
|
176
|
+
if escaped
|
|
177
|
+
escaped = false
|
|
178
|
+
elsif character == '\\'
|
|
179
|
+
escaped = true
|
|
180
|
+
elsif character == quote
|
|
181
|
+
quote = nil
|
|
182
|
+
end
|
|
183
|
+
elsif ["'", '"'].include?(character)
|
|
184
|
+
quote = character
|
|
185
|
+
elsif character == '('
|
|
186
|
+
deltas << 1
|
|
187
|
+
elsif character == ')'
|
|
188
|
+
deltas << -1
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
deltas
|
|
192
|
+
end
|
|
193
|
+
|
|
145
194
|
def locate_key_line(lines, start_index, end_index, key)
|
|
146
195
|
(start_index..end_index).find do |index|
|
|
147
196
|
lines[index].include?("\"#{key}\"") || lines[index].include?("@\"#{key}\"")
|
|
@@ -149,7 +198,7 @@ module I18nContextGenerator
|
|
|
149
198
|
end
|
|
150
199
|
|
|
151
200
|
def discover_android_entries(file)
|
|
152
|
-
lines =
|
|
201
|
+
lines = searchable_file_lines(file)
|
|
153
202
|
entries = []
|
|
154
203
|
|
|
155
204
|
lines.each_with_index do |line, index|
|
|
@@ -160,27 +209,23 @@ module I18nContextGenerator
|
|
|
160
209
|
end
|
|
161
210
|
|
|
162
211
|
def extract_android_entries_from_line(file, index, line)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
212
|
+
@localization_syntax.android_discovery_patterns.flat_map do |resource_type, pattern|
|
|
213
|
+
line.scan(pattern).filter_map do |captures|
|
|
214
|
+
key = captures.compact.first
|
|
215
|
+
next if key.nil? || key.empty?
|
|
216
|
+
|
|
217
|
+
DiscoveredLocalization.new(
|
|
218
|
+
key: key,
|
|
219
|
+
file: file,
|
|
220
|
+
line: index + 1,
|
|
221
|
+
resource_type: resource_type
|
|
222
|
+
)
|
|
223
|
+
end
|
|
172
224
|
end
|
|
173
225
|
end
|
|
174
226
|
|
|
175
227
|
def unescape_source_string(text)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
text
|
|
179
|
-
.gsub('\\"', '"')
|
|
180
|
-
.gsub("\\'", "'")
|
|
181
|
-
.gsub('\\\\', '\\')
|
|
182
|
-
.gsub('\\n', "\n")
|
|
183
|
-
.gsub('\\t', "\t")
|
|
228
|
+
AppleStringLiteral.decode(text)
|
|
184
229
|
end
|
|
185
230
|
end
|
|
186
231
|
end
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'concurrent'
|
|
4
|
+
require_relative 'path_policy'
|
|
5
|
+
require_relative 'file_classifier'
|
|
6
|
+
require_relative 'localization_syntax'
|
|
7
|
+
require_relative 'searcher/comment_masking'
|
|
8
|
+
require_relative 'searcher/match_filtering'
|
|
4
9
|
require_relative 'searcher/source_discovery'
|
|
5
10
|
|
|
6
11
|
module I18nContextGenerator
|
|
7
12
|
# Finds where translation keys are used in iOS and Android source code.
|
|
8
13
|
class Searcher
|
|
14
|
+
include CommentMasking
|
|
15
|
+
include MatchFiltering
|
|
9
16
|
include SourceDiscovery
|
|
10
17
|
|
|
11
18
|
# Represents a code match with surrounding context
|
|
@@ -16,46 +23,35 @@ module I18nContextGenerator
|
|
|
16
23
|
end
|
|
17
24
|
|
|
18
25
|
# Represents a localization entry discovered directly from source code.
|
|
19
|
-
DiscoveredLocalization = Data.define(
|
|
20
|
-
|
|
26
|
+
DiscoveredLocalization = Data.define(
|
|
27
|
+
:key, :file, :line, :text, :comment, :resource_type, :locations, :location_groups
|
|
28
|
+
) do
|
|
29
|
+
def initialize(key:, file:, line:, text: nil, comment: nil, resource_type: :string, locations: nil,
|
|
30
|
+
location_groups: nil)
|
|
31
|
+
locations ||= ["#{file}:#{line}"]
|
|
32
|
+
location_groups ||= [locations]
|
|
21
33
|
super
|
|
22
34
|
end
|
|
23
35
|
end
|
|
24
36
|
|
|
25
|
-
# Patterns that indicate false positive matches (not actual localization usage)
|
|
26
|
-
FALSE_POSITIVE_PATTERNS = [
|
|
27
|
-
/==\s*["']/, # String comparisons like == "yes"
|
|
28
|
-
/["']\s*==/, # String comparisons like "yes" ==
|
|
29
|
-
/!=\s*["']/, # String comparisons like != "no"
|
|
30
|
-
/["']\s*!=/, # String comparisons like "no" !=
|
|
31
|
-
/\.equals\(["']/, # Java .equals("string")
|
|
32
|
-
/contentEquals\(["']/ # Kotlin contentEquals
|
|
33
|
-
].freeze
|
|
34
|
-
|
|
35
|
-
# File extensions to search by platform
|
|
36
|
-
FILE_EXTENSIONS = {
|
|
37
|
-
ios: %w[.swift .m .mm .h].freeze,
|
|
38
|
-
android: %w[.kt .java .xml].freeze,
|
|
39
|
-
unknown: %w[.swift .m .mm .h .kt .java .xml].freeze
|
|
40
|
-
}.freeze
|
|
41
|
-
|
|
42
|
-
IOS_WRAPPER_DEFINITION_PATTERN =
|
|
43
|
-
/\b(static\s+)?(?:let|var)\s+(\w+)\s*=\s*(?:NSLocalizedString|String\s*\(\s*localized:|LocalizedStringKey\s*\(|Text\s*\()/
|
|
44
37
|
IOS_TYPE_DECLARATION_PATTERN = /\b(class|struct|enum|extension)\s+(\w+)/
|
|
45
38
|
|
|
46
|
-
def initialize(source_paths:, ignore_patterns:, context_lines: 15, platform: nil)
|
|
39
|
+
def initialize(source_paths:, ignore_patterns:, context_lines: 15, platform: nil, swift_functions: nil)
|
|
47
40
|
@source_paths = source_paths
|
|
48
|
-
@
|
|
41
|
+
@path_policy = PathPolicy.new(ignore_patterns: ignore_patterns, roots: source_paths)
|
|
49
42
|
@context_lines = context_lines
|
|
50
43
|
@platform = platform || detect_platform
|
|
44
|
+
@localization_syntax = LocalizationSyntax.new(swift_functions: swift_functions)
|
|
51
45
|
|
|
52
46
|
# Cache discovered files for repeated searches
|
|
53
47
|
@files_cache = nil
|
|
54
|
-
@
|
|
48
|
+
@files_cache_mutex = Mutex.new
|
|
49
|
+
@file_lines_cache = Concurrent::Map.new
|
|
50
|
+
@searchable_file_lines_cache = Concurrent::Map.new
|
|
55
51
|
end
|
|
56
52
|
|
|
57
|
-
def search(key)
|
|
58
|
-
patterns = build_search_patterns(key)
|
|
53
|
+
def search(key, resource_type: nil)
|
|
54
|
+
patterns = build_search_patterns(key, resource_type: resource_type)
|
|
59
55
|
files = discover_files
|
|
60
56
|
direct_matches = []
|
|
61
57
|
|
|
@@ -77,87 +73,47 @@ module I18nContextGenerator
|
|
|
77
73
|
|
|
78
74
|
def detect_platform
|
|
79
75
|
@source_paths.each do |path|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
Find.find(path) do |f|
|
|
84
|
-
if File.directory?(f) && ignored?(f)
|
|
85
|
-
Find.prune
|
|
86
|
-
next
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
next unless File.file?(f)
|
|
90
|
-
next if ignored?(f)
|
|
91
|
-
|
|
92
|
-
return :ios if f.end_with?('.swift', '.m', '.mm', '.h')
|
|
93
|
-
return :android if f.end_with?('.kt', '.java')
|
|
94
|
-
end
|
|
95
|
-
elsif !ignored?(path) && path.end_with?('.swift', '.m', '.mm', '.h')
|
|
96
|
-
return :ios
|
|
97
|
-
elsif !ignored?(path) && path.end_with?('.kt', '.java')
|
|
98
|
-
return :android
|
|
76
|
+
@path_policy.each_file(path) do |file|
|
|
77
|
+
platform = FileClassifier.source_platform(file)
|
|
78
|
+
return platform if platform
|
|
99
79
|
end
|
|
100
80
|
end
|
|
101
81
|
:unknown
|
|
102
82
|
end
|
|
103
83
|
|
|
104
|
-
def compile_ignore_patterns(patterns)
|
|
105
|
-
patterns.map { |p| glob_to_regex(p) }
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def glob_to_regex(glob_pattern)
|
|
109
|
-
# Convert glob pattern to regex
|
|
110
|
-
# Handle common glob patterns: *, **, ?
|
|
111
|
-
regex_str = Regexp.escape(glob_pattern)
|
|
112
|
-
.gsub('\*\*/', '(.*/)?') # **/ matches any path (including empty)
|
|
113
|
-
.gsub('\*\*', '.*') # ** matches anything
|
|
114
|
-
.gsub('\*', '[^/]*') # * matches within path segment
|
|
115
|
-
.gsub('\?', '.') # ? matches single char
|
|
116
|
-
Regexp.new("(?:^|/)#{regex_str}(?:$|/)")
|
|
117
|
-
end
|
|
118
|
-
|
|
119
84
|
def discover_files
|
|
120
85
|
return @files_cache if @files_cache
|
|
121
86
|
|
|
122
|
-
|
|
123
|
-
|
|
87
|
+
@files_cache_mutex.synchronize do
|
|
88
|
+
return @files_cache if @files_cache
|
|
124
89
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
files << path if extensions.any? { |ext| path.end_with?(ext) }
|
|
128
|
-
elsif File.directory?(path)
|
|
129
|
-
# Build a single glob pattern for all extensions
|
|
130
|
-
ext_pattern = extensions.size == 1 ? "*#{extensions.first}" : "*{#{extensions.join(',')}}"
|
|
131
|
-
files.concat(Dir.glob(File.join(path, '**', ext_pattern)))
|
|
90
|
+
@files_cache = @path_policy.files(@source_paths) do |file|
|
|
91
|
+
FileClassifier.searchable_source?(file, platform: @platform)
|
|
132
92
|
end
|
|
133
93
|
end
|
|
134
|
-
|
|
135
|
-
# Apply ignore patterns and cache
|
|
136
|
-
@files_cache = files.reject { |f| ignored?(f) }
|
|
137
94
|
end
|
|
138
95
|
|
|
139
|
-
def ignored?(file)
|
|
140
|
-
@
|
|
96
|
+
def ignored?(file, directory: false)
|
|
97
|
+
@path_policy.ignored?(file, directory: directory)
|
|
141
98
|
end
|
|
142
99
|
|
|
143
100
|
def search_file(file, patterns, key, enable_multiline: true)
|
|
144
101
|
matches = []
|
|
145
|
-
lines =
|
|
102
|
+
lines = cached_file_lines(file)
|
|
103
|
+
searchable_lines = searchable_file_lines(file)
|
|
146
104
|
match_indices = Set.new
|
|
147
105
|
|
|
148
|
-
#
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
lines << line
|
|
152
|
-
|
|
106
|
+
# Find matching line indices using the comment-masked source while
|
|
107
|
+
# retaining the original lines for prompt context and output.
|
|
108
|
+
searchable_lines.each_with_index do |line, index|
|
|
153
109
|
# Check if any pattern matches this line
|
|
154
110
|
match_indices << index if patterns.any? { |pattern| pattern.match?(line) }
|
|
155
111
|
end
|
|
156
112
|
|
|
157
113
|
# For iOS files, also check for multi-line NSLocalizedString patterns
|
|
158
114
|
# where the function call and key are on different lines
|
|
159
|
-
if enable_multiline && @platform == :ios &&
|
|
160
|
-
multiline_matches = find_multiline_ios_matches(
|
|
115
|
+
if enable_multiline && @platform == :ios && FileClassifier.searchable_platform(file) == :ios
|
|
116
|
+
multiline_matches = find_multiline_ios_matches(searchable_lines, patterns, key)
|
|
161
117
|
match_indices.merge(multiline_matches)
|
|
162
118
|
end
|
|
163
119
|
|
|
@@ -225,29 +181,35 @@ module I18nContextGenerator
|
|
|
225
181
|
return unless definition_index
|
|
226
182
|
|
|
227
183
|
definition_line = lines[definition_index]
|
|
228
|
-
definition_match =
|
|
229
|
-
return unless definition_match
|
|
184
|
+
definition_match = @localization_syntax.ios_wrapper_definition_pattern.match(definition_line)
|
|
185
|
+
return unless definition_match
|
|
230
186
|
|
|
231
187
|
type_path = find_ios_type_path(lines, definition_index)
|
|
232
188
|
return if type_path.empty?
|
|
233
189
|
|
|
234
190
|
{
|
|
235
191
|
type_path: type_path,
|
|
236
|
-
member_name: definition_match[
|
|
192
|
+
member_name: definition_match[:member_name],
|
|
237
193
|
definition_file: match.file,
|
|
238
194
|
definition_line: definition_index + 1
|
|
239
195
|
}
|
|
240
196
|
end
|
|
241
197
|
|
|
242
198
|
def cached_file_lines(file)
|
|
243
|
-
@file_lines_cache
|
|
199
|
+
@file_lines_cache.compute_if_absent(file) { File.readlines(file, chomp: true) }
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def searchable_file_lines(file)
|
|
203
|
+
@searchable_file_lines_cache.compute_if_absent(file) do
|
|
204
|
+
mask_comments(cached_file_lines(file), file)
|
|
205
|
+
end
|
|
244
206
|
end
|
|
245
207
|
|
|
246
208
|
def find_ios_wrapper_definition_index(lines, match_index, lookback: 5)
|
|
247
209
|
start_idx = [0, match_index - lookback].max
|
|
248
210
|
|
|
249
211
|
match_index.downto(start_idx) do |index|
|
|
250
|
-
return index if
|
|
212
|
+
return index if @localization_syntax.ios_wrapper_definition_pattern.match?(lines[index])
|
|
251
213
|
end
|
|
252
214
|
|
|
253
215
|
nil
|
|
@@ -292,22 +254,24 @@ module I18nContextGenerator
|
|
|
292
254
|
# Find matches where localization calls span multiple lines
|
|
293
255
|
# e.g., NSLocalizedString(\n "key",\n comment: "...")
|
|
294
256
|
def find_multiline_ios_matches(lines, patterns, key)
|
|
295
|
-
key_pattern =
|
|
257
|
+
key_pattern = /@?"#{LocalizationSyntax.swift_string_content_pattern(key)}"/
|
|
258
|
+
call_patterns = @localization_syntax.ios_multiline_search_patterns(key)
|
|
296
259
|
|
|
297
260
|
lines.each_with_index.filter_map do |line, index|
|
|
298
261
|
next if patterns.any? { |p| p.match?(line) } # Already a single-line match
|
|
299
262
|
next unless key_pattern.match?(line) # Doesn't contain the key
|
|
300
263
|
|
|
301
|
-
index if
|
|
264
|
+
index if preceded_by_localization_call?(lines, index, call_patterns)
|
|
302
265
|
end.to_set
|
|
303
266
|
end
|
|
304
267
|
|
|
305
|
-
def
|
|
268
|
+
def preceded_by_localization_call?(lines, index, patterns, lookback: 5)
|
|
306
269
|
start_idx = [0, index - lookback].max
|
|
307
270
|
|
|
308
271
|
(start_idx...index).reverse_each do |i|
|
|
309
272
|
line = lines[i]
|
|
310
|
-
|
|
273
|
+
snippet = lines[i..index].join("\n")
|
|
274
|
+
return true if patterns.any? { |pattern| pattern.match?(snippet) }
|
|
311
275
|
return false if line =~ /;\s*$/ || line =~ /\)\s*$/ # Hit a statement boundary
|
|
312
276
|
end
|
|
313
277
|
|
|
@@ -336,124 +300,16 @@ module I18nContextGenerator
|
|
|
336
300
|
context_parts.join("\n")
|
|
337
301
|
end
|
|
338
302
|
|
|
339
|
-
def build_search_patterns(key)
|
|
340
|
-
|
|
341
|
-
when :ios
|
|
342
|
-
build_ios_patterns(key)
|
|
343
|
-
when :android
|
|
344
|
-
build_android_patterns(key)
|
|
345
|
-
else
|
|
346
|
-
build_ios_patterns(key) + build_android_patterns(key) + [Regexp.escape(key)]
|
|
347
|
-
end
|
|
348
|
-
|
|
349
|
-
# Pre-compile all patterns for this search
|
|
350
|
-
pattern_strings.map { |p| Regexp.new(p) }
|
|
351
|
-
end
|
|
352
|
-
|
|
353
|
-
# Extract the base resource name from composite Android keys
|
|
354
|
-
# e.g., "post_likes_count:one" -> "post_likes_count"
|
|
355
|
-
# "days_of_week[0]" -> "days_of_week"
|
|
356
|
-
def android_base_key(key)
|
|
357
|
-
key.sub(/:[a-z]+$/, '').sub(/\[\d+\]$/, '')
|
|
303
|
+
def build_search_patterns(key, resource_type: nil)
|
|
304
|
+
@localization_syntax.search_patterns(key, platform: @platform, resource_type: resource_type)
|
|
358
305
|
end
|
|
359
306
|
|
|
360
|
-
# Patterns that indicate the start of a localization function call
|
|
361
|
-
# Used for multi-line matching when the key is on a different line
|
|
362
|
-
IOS_FUNCTION_OPENERS = [
|
|
363
|
-
/NSLocalizedString\s*\(\s*$/,
|
|
364
|
-
/String\s*\(\s*localized:\s*$/,
|
|
365
|
-
/LocalizedStringKey\s*\(\s*$/,
|
|
366
|
-
/Text\s*\(\s*$/
|
|
367
|
-
].freeze
|
|
368
|
-
private_constant :IOS_FUNCTION_OPENERS
|
|
369
|
-
|
|
370
307
|
def build_ios_patterns(key)
|
|
371
|
-
|
|
372
|
-
[
|
|
373
|
-
# NSLocalizedString("key", ...) - most common (Swift and Obj-C)
|
|
374
|
-
# Note: @? handles optional @ prefix for Objective-C @"string" syntax
|
|
375
|
-
"NSLocalizedString\\s*\\(\\s*@?[\"']#{escaped}[\"']",
|
|
376
|
-
# String(localized: "key", ...) - modern Swift
|
|
377
|
-
"String\\s*\\(\\s*localized:\\s*[\"']#{escaped}[\"']",
|
|
378
|
-
# LocalizedStringKey("key") - SwiftUI
|
|
379
|
-
"LocalizedStringKey\\s*\\(\\s*[\"']#{escaped}[\"']",
|
|
380
|
-
# Direct assignment to a LocalizedStringKey-typed value
|
|
381
|
-
"LocalizedStringKey\\s*=\\s*[\"']#{escaped}[\"']",
|
|
382
|
-
# Text("key") - SwiftUI (when using localized strings)
|
|
383
|
-
"Text\\s*\\(\\s*[\"']#{escaped}[\"']",
|
|
384
|
-
# .localized extension pattern
|
|
385
|
-
"[\"']#{escaped}[\"']\\.localized"
|
|
386
|
-
]
|
|
387
|
-
end
|
|
388
|
-
|
|
389
|
-
def build_android_patterns(key)
|
|
390
|
-
base = android_base_key(key)
|
|
391
|
-
escaped_base = Regexp.escape(base)
|
|
392
|
-
|
|
393
|
-
if key =~ /:[a-z]+$/
|
|
394
|
-
# Plural key (e.g., "post_likes_count:one") — search by base name in plural resources
|
|
395
|
-
[
|
|
396
|
-
"R\\.plurals\\.#{escaped_base}\\b",
|
|
397
|
-
"@plurals/#{escaped_base}\\b",
|
|
398
|
-
"getQuantityString\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
|
|
399
|
-
"\\.getQuantityString\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
|
|
400
|
-
"pluralStringResource\\s*\\(\\s*R\\.plurals\\.#{escaped_base}",
|
|
401
|
-
"[\\(\\s,=]plurals\\.#{escaped_base}\\b"
|
|
402
|
-
]
|
|
403
|
-
elsif key =~ /\[\d+\]$/
|
|
404
|
-
# Array key (e.g., "days_of_week[0]") — search by base name in array resources
|
|
405
|
-
[
|
|
406
|
-
"R\\.array\\.#{escaped_base}\\b",
|
|
407
|
-
"@array/#{escaped_base}\\b",
|
|
408
|
-
"getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
|
|
409
|
-
"\\.getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
|
|
410
|
-
"resources\\.getStringArray\\s*\\(\\s*R\\.array\\.#{escaped_base}",
|
|
411
|
-
"[\\(\\s,=]array\\.#{escaped_base}\\b"
|
|
412
|
-
]
|
|
413
|
-
else
|
|
414
|
-
# Standard string key
|
|
415
|
-
escaped = Regexp.escape(key)
|
|
416
|
-
[
|
|
417
|
-
"R\\.string\\.#{escaped}\\b",
|
|
418
|
-
"@string/#{escaped}\\b",
|
|
419
|
-
"getString\\s*\\(\\s*R\\.string\\.#{escaped}",
|
|
420
|
-
"\\.getString\\s*\\(\\s*R\\.string\\.#{escaped}",
|
|
421
|
-
"stringResource\\s*\\(\\s*R\\.string\\.#{escaped}",
|
|
422
|
-
"[\\(\\s,=]string\\.#{escaped}\\b",
|
|
423
|
-
"getString\\s*\\(\\s*string\\.#{escaped}",
|
|
424
|
-
"stringResource\\s*\\(\\s*string\\.#{escaped}"
|
|
425
|
-
]
|
|
426
|
-
end
|
|
308
|
+
@localization_syntax.ios_search_patterns(key)
|
|
427
309
|
end
|
|
428
310
|
|
|
429
|
-
def
|
|
430
|
-
|
|
431
|
-
matches.select do |match|
|
|
432
|
-
location = "#{match.file}:#{match.line}"
|
|
433
|
-
next false if seen.include?(location)
|
|
434
|
-
next false if false_positive?(match.match_line, key)
|
|
435
|
-
next false if translation_file?(match.file)
|
|
436
|
-
|
|
437
|
-
seen.add(location)
|
|
438
|
-
end
|
|
439
|
-
end
|
|
440
|
-
|
|
441
|
-
def false_positive?(line, _key)
|
|
442
|
-
return false if line.nil? || line.empty?
|
|
443
|
-
|
|
444
|
-
FALSE_POSITIVE_PATTERNS.any? { |pattern| pattern.match?(line) }
|
|
445
|
-
end
|
|
446
|
-
|
|
447
|
-
def translation_file?(file)
|
|
448
|
-
basename = File.basename(file).downcase
|
|
449
|
-
ext = File.extname(file).downcase
|
|
450
|
-
|
|
451
|
-
# Skip translation files - we want code usage, not definitions
|
|
452
|
-
return true if ext == '.strings'
|
|
453
|
-
return true if basename == 'strings.xml'
|
|
454
|
-
return true if file.include?('/res/values') && ext == '.xml'
|
|
455
|
-
|
|
456
|
-
false
|
|
311
|
+
def build_android_patterns(key, resource_type: nil)
|
|
312
|
+
@localization_syntax.android_search_patterns(key, resource_type: resource_type)
|
|
457
313
|
end
|
|
458
314
|
end
|
|
459
315
|
end
|