i18n-context-generator 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +163 -26
- data/lib/i18n_context_generator/android_resource.rb +209 -0
- data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
- data/lib/i18n_context_generator/cache.rb +40 -8
- data/lib/i18n_context_generator/changed_location.rb +55 -0
- data/lib/i18n_context_generator/cli.rb +163 -76
- data/lib/i18n_context_generator/config/cli_values.rb +33 -0
- data/lib/i18n_context_generator/config/defaults.rb +35 -0
- data/lib/i18n_context_generator/config/schema.rb +225 -0
- data/lib/i18n_context_generator/config/serialization.rb +64 -0
- data/lib/i18n_context_generator/config/validation.rb +249 -0
- data/lib/i18n_context_generator/config.rb +293 -102
- data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
- data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
- data/lib/i18n_context_generator/context_extractor/run_logging.rb +45 -6
- data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
- data/lib/i18n_context_generator/context_extractor/source_filters.rb +51 -5
- data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
- data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
- data/lib/i18n_context_generator/context_extractor.rb +207 -216
- data/lib/i18n_context_generator/file_classifier.rb +72 -0
- data/lib/i18n_context_generator/generated_comment.rb +32 -0
- data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
- data/lib/i18n_context_generator/git_diff.rb +224 -95
- data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
- data/lib/i18n_context_generator/llm/client.rb +304 -92
- data/lib/i18n_context_generator/llm/openai.rb +92 -51
- data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
- data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
- data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
- data/lib/i18n_context_generator/localization_syntax.rb +246 -0
- data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
- data/lib/i18n_context_generator/parsers/base.rb +7 -4
- data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
- data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
- data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
- data/lib/i18n_context_generator/path_policy.rb +107 -0
- data/lib/i18n_context_generator/platform_validator.rb +24 -50
- data/lib/i18n_context_generator/run_metrics.rb +47 -0
- data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
- data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
- data/lib/i18n_context_generator/searcher/source_discovery.rb +109 -64
- data/lib/i18n_context_generator/searcher.rb +61 -205
- data/lib/i18n_context_generator/supplemental_context.rb +77 -0
- data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
- data/lib/i18n_context_generator/version.rb +1 -1
- data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
- data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
- data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
- data/lib/i18n_context_generator/writers/helpers.rb +7 -29
- data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
- data/lib/i18n_context_generator/writers/preview.rb +80 -0
- data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
- data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
- data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
- data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
- data/lib/i18n_context_generator/xcstrings_document.rb +248 -0
- data/lib/i18n_context_generator/xml_scanner.rb +38 -0
- data/lib/i18n_context_generator.rb +20 -4
- metadata +59 -12
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module I18nContextGenerator
|
|
4
|
+
# Shared ownership and merge rules for context generated into destination
|
|
5
|
+
# comments. Writers remain responsible for destination-specific escaping.
|
|
6
|
+
module GeneratedComment
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def merge(existing:, generated:, prefix:, mode:, separator:)
|
|
10
|
+
return generated if existing.nil? || existing.empty? || mode.to_s == 'replace'
|
|
11
|
+
|
|
12
|
+
if prefix.empty?
|
|
13
|
+
return existing if existing.include?(generated)
|
|
14
|
+
|
|
15
|
+
return "#{existing}#{separator}#{generated}"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
return replace_managed(existing, generated, prefix) if managed?(existing, prefix: prefix)
|
|
19
|
+
|
|
20
|
+
"#{existing}#{separator}#{generated}"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def managed?(comment, prefix:)
|
|
24
|
+
prefix.empty? || comment.to_s.include?(prefix)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def replace_managed(existing, generated, prefix)
|
|
28
|
+
existing.gsub(/#{Regexp.escape(prefix)}[^\n]*/) { generated }
|
|
29
|
+
end
|
|
30
|
+
private_class_method :replace_managed
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../android_resource'
|
|
4
|
+
require_relative '../translation_comment_index'
|
|
5
|
+
require_relative '../xml_scanner'
|
|
6
|
+
|
|
7
|
+
module I18nContextGenerator
|
|
8
|
+
# Android XML diff parsing kept separate from Git command/path orchestration.
|
|
9
|
+
module GitDiffXmlChanges
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def extract_xml_key_locations(diff_output, file_path, base_content: nil, head_content: nil)
|
|
13
|
+
extract_xml_changes(
|
|
14
|
+
diff_output,
|
|
15
|
+
file_path,
|
|
16
|
+
base_content: base_content,
|
|
17
|
+
head_content: head_content
|
|
18
|
+
)[:locations]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def extract_xml_changes(diff_output, file_path, base_content: nil, head_content: nil)
|
|
22
|
+
state = initial_xml_diff_state(file_path)
|
|
23
|
+
parse_xml_diff!(state, diff_output)
|
|
24
|
+
resolve_orphaned_items(
|
|
25
|
+
state[:keys],
|
|
26
|
+
state[:orphaned_item_file_lines],
|
|
27
|
+
file_path,
|
|
28
|
+
locations: state[:locations],
|
|
29
|
+
content: head_content
|
|
30
|
+
)
|
|
31
|
+
add_comment_only_locations(
|
|
32
|
+
state[:locations],
|
|
33
|
+
state[:added_file_lines],
|
|
34
|
+
file_path,
|
|
35
|
+
format: :xml,
|
|
36
|
+
content: head_content
|
|
37
|
+
) { |line_number| line_number }
|
|
38
|
+
|
|
39
|
+
typed_locations = state[:locations].transform_values do |lines|
|
|
40
|
+
lines.map { |line| changed_location(file_path, line, side: :right) }
|
|
41
|
+
end
|
|
42
|
+
merge_removed_xml_locations!(
|
|
43
|
+
typed_locations,
|
|
44
|
+
diff_output,
|
|
45
|
+
file_path,
|
|
46
|
+
base_content: base_content,
|
|
47
|
+
head_content: head_content
|
|
48
|
+
)
|
|
49
|
+
typed_locations = prefer_right_locations(typed_locations)
|
|
50
|
+
state[:keys].merge(typed_locations.keys)
|
|
51
|
+
|
|
52
|
+
{ keys: state[:keys], locations: typed_locations }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def initial_xml_diff_state(file_path)
|
|
56
|
+
{
|
|
57
|
+
keys: Set.new,
|
|
58
|
+
locations: Hash.new { |hash, key| hash[key] = Set.new },
|
|
59
|
+
current_parent: nil,
|
|
60
|
+
current_string: nil,
|
|
61
|
+
file_line: nil,
|
|
62
|
+
orphaned_item_file_lines: [],
|
|
63
|
+
pending_tag: nil,
|
|
64
|
+
file_path: file_path,
|
|
65
|
+
added_file_lines: [],
|
|
66
|
+
xml_comment_state: {}
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def parse_xml_diff!(state, diff_output)
|
|
71
|
+
diff_output.each_line do |line|
|
|
72
|
+
next if update_xml_hunk_line?(state, line)
|
|
73
|
+
next if line.start_with?('diff ', 'index ', '--- ', '+++ ')
|
|
74
|
+
|
|
75
|
+
is_removed = line.start_with?('-')
|
|
76
|
+
is_added = line.start_with?('+')
|
|
77
|
+
content = line.sub(/^[ +-]/, '')
|
|
78
|
+
process_xml_diff_line(state, content, added: is_added) unless is_removed
|
|
79
|
+
state[:file_line] += 1 if state[:file_line] && !is_removed
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def process_xml_diff_line(state, content, added:)
|
|
84
|
+
state[:added_file_lines] << state[:file_line] if added && state[:file_line]
|
|
85
|
+
visible_content, contained_comment = XmlScanner.without_comments(
|
|
86
|
+
content,
|
|
87
|
+
state[:xml_comment_state]
|
|
88
|
+
)
|
|
89
|
+
process_xml_diff_content(
|
|
90
|
+
state,
|
|
91
|
+
visible_content,
|
|
92
|
+
added: added,
|
|
93
|
+
contained_comment: contained_comment
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def update_xml_hunk_line?(state, line)
|
|
98
|
+
hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/)
|
|
99
|
+
return false unless hunk
|
|
100
|
+
|
|
101
|
+
state[:file_line] = hunk[1].to_i
|
|
102
|
+
state[:current_parent] = nil
|
|
103
|
+
state[:current_string] = nil
|
|
104
|
+
state[:pending_tag] = nil
|
|
105
|
+
state[:xml_comment_state] = {}
|
|
106
|
+
true
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def process_xml_diff_content(state, content, added:, contained_comment: false)
|
|
110
|
+
meaningful_change = !content.strip.empty? || contained_comment
|
|
111
|
+
record_xml_change(state, state[:current_parent]) if added && meaningful_change && state[:current_parent]
|
|
112
|
+
record_xml_change(state, state[:current_string]) if added && meaningful_change && state[:current_string]
|
|
113
|
+
|
|
114
|
+
accumulate_xml_opening_tag(state, content, added: added)
|
|
115
|
+
complete_xml_opening_tag(state) if state.dig(:pending_tag, :content)&.include?('>')
|
|
116
|
+
track_xml_item_change(state) if added && content.match?(/<item\b/)
|
|
117
|
+
|
|
118
|
+
state[:current_string] = nil if content.match?(%r{</string>})
|
|
119
|
+
state[:current_parent] = nil if content.match?(%r{</(?:plurals|string-array)>})
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def accumulate_xml_opening_tag(state, content, added:)
|
|
123
|
+
if state[:pending_tag]
|
|
124
|
+
state[:pending_tag][:content] << content
|
|
125
|
+
state[:pending_tag][:added] ||= added
|
|
126
|
+
state[:pending_tag][:first_added_line] ||= state[:file_line] if added
|
|
127
|
+
elsif (tag_start = content.index(/<(?:string-array|plurals|string)\b/))
|
|
128
|
+
state[:pending_tag] = {
|
|
129
|
+
content: content[tag_start..],
|
|
130
|
+
added: added,
|
|
131
|
+
first_added_line: (state[:file_line] if added)
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def complete_xml_opening_tag(state)
|
|
137
|
+
tag = state[:pending_tag]
|
|
138
|
+
resource = resource_from_opening_tag(tag[:content])
|
|
139
|
+
if resource
|
|
140
|
+
record_xml_change(state, resource[:name], line: tag[:first_added_line]) if tag[:added]
|
|
141
|
+
track_open_xml_resource(state, resource, tag[:content])
|
|
142
|
+
end
|
|
143
|
+
state[:pending_tag] = nil
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def track_open_xml_resource(state, resource, content)
|
|
147
|
+
if resource[:type] == :string
|
|
148
|
+
state[:current_string] = resource[:name] unless content.include?('</string>')
|
|
149
|
+
elsif !content.include?("</#{AndroidResource.tag_for_type(resource[:type])}>")
|
|
150
|
+
state[:current_parent] = resource[:name]
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def track_xml_item_change(state)
|
|
155
|
+
if state[:current_parent]
|
|
156
|
+
record_xml_change(state, state[:current_parent])
|
|
157
|
+
elsif state[:file_line]
|
|
158
|
+
state[:orphaned_item_file_lines] << state[:file_line]
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def record_xml_change(state, key, line: state[:file_line])
|
|
163
|
+
return unless key
|
|
164
|
+
|
|
165
|
+
state[:keys] << key
|
|
166
|
+
state[:locations][key] << line if line
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def resource_from_opening_tag(tag)
|
|
170
|
+
type_match = tag.match(/<(string-array|plurals|string)\b/)
|
|
171
|
+
name_match = tag.match(/\bname\s*=\s*(["'])(.*?)\1/m)
|
|
172
|
+
return unless type_match && name_match
|
|
173
|
+
|
|
174
|
+
{ type: AndroidResource.type_for_tag(type_match[1]), name: name_match[2] }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def resolve_orphaned_items(keys, orphaned_lines, file_path, locations: nil, content: nil)
|
|
178
|
+
return if orphaned_lines.empty?
|
|
179
|
+
return if content.nil? && !File.exist?(file_path)
|
|
180
|
+
|
|
181
|
+
resource_index = AndroidResource.index(content || File.read(file_path, encoding: 'UTF-8'))
|
|
182
|
+
|
|
183
|
+
orphaned_lines.each do |line_num|
|
|
184
|
+
parent = resource_index.base_key_at(line_num)
|
|
185
|
+
next unless parent
|
|
186
|
+
|
|
187
|
+
keys << parent
|
|
188
|
+
locations[parent] << line_num if locations
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def add_comment_only_locations(locations, added_lines, file_path, format:, content: nil)
|
|
193
|
+
return if content.nil? && !File.file?(file_path)
|
|
194
|
+
|
|
195
|
+
comment_index = TranslationCommentIndex.new(file_path, format: format, content: content)
|
|
196
|
+
|
|
197
|
+
added_lines.each do |line_number|
|
|
198
|
+
key = comment_index.key_at(line_number)
|
|
199
|
+
next unless key
|
|
200
|
+
next if locations[key].any?
|
|
201
|
+
|
|
202
|
+
locations[key] << yield(line_number)
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def merge_removed_xml_locations!(locations, diff_output, file_path, base_content:, head_content:)
|
|
207
|
+
return unless base_content && head_content
|
|
208
|
+
|
|
209
|
+
base_comments = TranslationCommentIndex.new(format: :xml, content: base_content)
|
|
210
|
+
head_comments = TranslationCommentIndex.new(format: :xml, content: head_content)
|
|
211
|
+
base_resources = AndroidResource.index(base_content)
|
|
212
|
+
head_resources = AndroidResource.index(head_content)
|
|
213
|
+
|
|
214
|
+
each_changed_diff_line(diff_output) do |content, old_line, _new_line, side|
|
|
215
|
+
next unless side == :left
|
|
216
|
+
next if content.strip.empty?
|
|
217
|
+
|
|
218
|
+
key = base_comments.key_at(old_line) || base_resources.base_key_at(old_line)
|
|
219
|
+
next unless key
|
|
220
|
+
|
|
221
|
+
fallback_line = head_comments.line_for_key(key) || resource_start_line(head_resources, key)
|
|
222
|
+
(locations[key] ||= []) << changed_location(
|
|
223
|
+
file_path,
|
|
224
|
+
old_line,
|
|
225
|
+
side: :left,
|
|
226
|
+
fallback_line: fallback_line
|
|
227
|
+
)
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def resource_start_line(index, key)
|
|
232
|
+
index.resource_spans.find { |span| span.base_key == key }&.start_line
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
end
|