i18n-context-generator 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +163 -26
  3. data/lib/i18n_context_generator/android_resource.rb +209 -0
  4. data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
  5. data/lib/i18n_context_generator/cache.rb +40 -8
  6. data/lib/i18n_context_generator/changed_location.rb +55 -0
  7. data/lib/i18n_context_generator/cli.rb +163 -76
  8. data/lib/i18n_context_generator/config/cli_values.rb +33 -0
  9. data/lib/i18n_context_generator/config/defaults.rb +35 -0
  10. data/lib/i18n_context_generator/config/schema.rb +225 -0
  11. data/lib/i18n_context_generator/config/serialization.rb +64 -0
  12. data/lib/i18n_context_generator/config/validation.rb +249 -0
  13. data/lib/i18n_context_generator/config.rb +293 -102
  14. data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
  15. data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
  16. data/lib/i18n_context_generator/context_extractor/run_logging.rb +45 -6
  17. data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
  18. data/lib/i18n_context_generator/context_extractor/source_filters.rb +51 -5
  19. data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
  20. data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
  21. data/lib/i18n_context_generator/context_extractor.rb +207 -216
  22. data/lib/i18n_context_generator/file_classifier.rb +72 -0
  23. data/lib/i18n_context_generator/generated_comment.rb +32 -0
  24. data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
  25. data/lib/i18n_context_generator/git_diff.rb +224 -95
  26. data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
  27. data/lib/i18n_context_generator/llm/client.rb +304 -92
  28. data/lib/i18n_context_generator/llm/openai.rb +92 -51
  29. data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
  30. data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
  31. data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
  32. data/lib/i18n_context_generator/localization_syntax.rb +246 -0
  33. data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
  34. data/lib/i18n_context_generator/parsers/base.rb +7 -4
  35. data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
  36. data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
  37. data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
  38. data/lib/i18n_context_generator/path_policy.rb +107 -0
  39. data/lib/i18n_context_generator/platform_validator.rb +24 -50
  40. data/lib/i18n_context_generator/run_metrics.rb +47 -0
  41. data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
  42. data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
  43. data/lib/i18n_context_generator/searcher/source_discovery.rb +109 -64
  44. data/lib/i18n_context_generator/searcher.rb +61 -205
  45. data/lib/i18n_context_generator/supplemental_context.rb +77 -0
  46. data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
  47. data/lib/i18n_context_generator/version.rb +1 -1
  48. data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
  49. data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
  50. data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
  51. data/lib/i18n_context_generator/writers/helpers.rb +7 -29
  52. data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
  53. data/lib/i18n_context_generator/writers/preview.rb +80 -0
  54. data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
  55. data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
  56. data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
  57. data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
  58. data/lib/i18n_context_generator/xcstrings_document.rb +248 -0
  59. data/lib/i18n_context_generator/xml_scanner.rb +38 -0
  60. data/lib/i18n_context_generator.rb +20 -4
  61. metadata +59 -12
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ # Shared ownership and merge rules for context generated into destination
5
+ # comments. Writers remain responsible for destination-specific escaping.
6
+ module GeneratedComment
7
+ module_function
8
+
9
+ def merge(existing:, generated:, prefix:, mode:, separator:)
10
+ return generated if existing.nil? || existing.empty? || mode.to_s == 'replace'
11
+
12
+ if prefix.empty?
13
+ return existing if existing.include?(generated)
14
+
15
+ return "#{existing}#{separator}#{generated}"
16
+ end
17
+
18
+ return replace_managed(existing, generated, prefix) if managed?(existing, prefix: prefix)
19
+
20
+ "#{existing}#{separator}#{generated}"
21
+ end
22
+
23
+ def managed?(comment, prefix:)
24
+ prefix.empty? || comment.to_s.include?(prefix)
25
+ end
26
+
27
+ def replace_managed(existing, generated, prefix)
28
+ existing.gsub(/#{Regexp.escape(prefix)}[^\n]*/) { generated }
29
+ end
30
+ private_class_method :replace_managed
31
+ end
32
+ end
@@ -0,0 +1,235 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../android_resource'
4
+ require_relative '../translation_comment_index'
5
+ require_relative '../xml_scanner'
6
+
7
+ module I18nContextGenerator
8
+ # Android XML diff parsing kept separate from Git command/path orchestration.
9
+ module GitDiffXmlChanges
10
+ private
11
+
12
+ def extract_xml_key_locations(diff_output, file_path, base_content: nil, head_content: nil)
13
+ extract_xml_changes(
14
+ diff_output,
15
+ file_path,
16
+ base_content: base_content,
17
+ head_content: head_content
18
+ )[:locations]
19
+ end
20
+
21
+ def extract_xml_changes(diff_output, file_path, base_content: nil, head_content: nil)
22
+ state = initial_xml_diff_state(file_path)
23
+ parse_xml_diff!(state, diff_output)
24
+ resolve_orphaned_items(
25
+ state[:keys],
26
+ state[:orphaned_item_file_lines],
27
+ file_path,
28
+ locations: state[:locations],
29
+ content: head_content
30
+ )
31
+ add_comment_only_locations(
32
+ state[:locations],
33
+ state[:added_file_lines],
34
+ file_path,
35
+ format: :xml,
36
+ content: head_content
37
+ ) { |line_number| line_number }
38
+
39
+ typed_locations = state[:locations].transform_values do |lines|
40
+ lines.map { |line| changed_location(file_path, line, side: :right) }
41
+ end
42
+ merge_removed_xml_locations!(
43
+ typed_locations,
44
+ diff_output,
45
+ file_path,
46
+ base_content: base_content,
47
+ head_content: head_content
48
+ )
49
+ typed_locations = prefer_right_locations(typed_locations)
50
+ state[:keys].merge(typed_locations.keys)
51
+
52
+ { keys: state[:keys], locations: typed_locations }
53
+ end
54
+
55
+ def initial_xml_diff_state(file_path)
56
+ {
57
+ keys: Set.new,
58
+ locations: Hash.new { |hash, key| hash[key] = Set.new },
59
+ current_parent: nil,
60
+ current_string: nil,
61
+ file_line: nil,
62
+ orphaned_item_file_lines: [],
63
+ pending_tag: nil,
64
+ file_path: file_path,
65
+ added_file_lines: [],
66
+ xml_comment_state: {}
67
+ }
68
+ end
69
+
70
+ def parse_xml_diff!(state, diff_output)
71
+ diff_output.each_line do |line|
72
+ next if update_xml_hunk_line?(state, line)
73
+ next if line.start_with?('diff ', 'index ', '--- ', '+++ ')
74
+
75
+ is_removed = line.start_with?('-')
76
+ is_added = line.start_with?('+')
77
+ content = line.sub(/^[ +-]/, '')
78
+ process_xml_diff_line(state, content, added: is_added) unless is_removed
79
+ state[:file_line] += 1 if state[:file_line] && !is_removed
80
+ end
81
+ end
82
+
83
+ def process_xml_diff_line(state, content, added:)
84
+ state[:added_file_lines] << state[:file_line] if added && state[:file_line]
85
+ visible_content, contained_comment = XmlScanner.without_comments(
86
+ content,
87
+ state[:xml_comment_state]
88
+ )
89
+ process_xml_diff_content(
90
+ state,
91
+ visible_content,
92
+ added: added,
93
+ contained_comment: contained_comment
94
+ )
95
+ end
96
+
97
+ def update_xml_hunk_line?(state, line)
98
+ hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/)
99
+ return false unless hunk
100
+
101
+ state[:file_line] = hunk[1].to_i
102
+ state[:current_parent] = nil
103
+ state[:current_string] = nil
104
+ state[:pending_tag] = nil
105
+ state[:xml_comment_state] = {}
106
+ true
107
+ end
108
+
109
+ def process_xml_diff_content(state, content, added:, contained_comment: false)
110
+ meaningful_change = !content.strip.empty? || contained_comment
111
+ record_xml_change(state, state[:current_parent]) if added && meaningful_change && state[:current_parent]
112
+ record_xml_change(state, state[:current_string]) if added && meaningful_change && state[:current_string]
113
+
114
+ accumulate_xml_opening_tag(state, content, added: added)
115
+ complete_xml_opening_tag(state) if state.dig(:pending_tag, :content)&.include?('>')
116
+ track_xml_item_change(state) if added && content.match?(/<item\b/)
117
+
118
+ state[:current_string] = nil if content.match?(%r{</string>})
119
+ state[:current_parent] = nil if content.match?(%r{</(?:plurals|string-array)>})
120
+ end
121
+
122
+ def accumulate_xml_opening_tag(state, content, added:)
123
+ if state[:pending_tag]
124
+ state[:pending_tag][:content] << content
125
+ state[:pending_tag][:added] ||= added
126
+ state[:pending_tag][:first_added_line] ||= state[:file_line] if added
127
+ elsif (tag_start = content.index(/<(?:string-array|plurals|string)\b/))
128
+ state[:pending_tag] = {
129
+ content: content[tag_start..],
130
+ added: added,
131
+ first_added_line: (state[:file_line] if added)
132
+ }
133
+ end
134
+ end
135
+
136
+ def complete_xml_opening_tag(state)
137
+ tag = state[:pending_tag]
138
+ resource = resource_from_opening_tag(tag[:content])
139
+ if resource
140
+ record_xml_change(state, resource[:name], line: tag[:first_added_line]) if tag[:added]
141
+ track_open_xml_resource(state, resource, tag[:content])
142
+ end
143
+ state[:pending_tag] = nil
144
+ end
145
+
146
+ def track_open_xml_resource(state, resource, content)
147
+ if resource[:type] == :string
148
+ state[:current_string] = resource[:name] unless content.include?('</string>')
149
+ elsif !content.include?("</#{AndroidResource.tag_for_type(resource[:type])}>")
150
+ state[:current_parent] = resource[:name]
151
+ end
152
+ end
153
+
154
+ def track_xml_item_change(state)
155
+ if state[:current_parent]
156
+ record_xml_change(state, state[:current_parent])
157
+ elsif state[:file_line]
158
+ state[:orphaned_item_file_lines] << state[:file_line]
159
+ end
160
+ end
161
+
162
+ def record_xml_change(state, key, line: state[:file_line])
163
+ return unless key
164
+
165
+ state[:keys] << key
166
+ state[:locations][key] << line if line
167
+ end
168
+
169
+ def resource_from_opening_tag(tag)
170
+ type_match = tag.match(/<(string-array|plurals|string)\b/)
171
+ name_match = tag.match(/\bname\s*=\s*(["'])(.*?)\1/m)
172
+ return unless type_match && name_match
173
+
174
+ { type: AndroidResource.type_for_tag(type_match[1]), name: name_match[2] }
175
+ end
176
+
177
+ def resolve_orphaned_items(keys, orphaned_lines, file_path, locations: nil, content: nil)
178
+ return if orphaned_lines.empty?
179
+ return if content.nil? && !File.exist?(file_path)
180
+
181
+ resource_index = AndroidResource.index(content || File.read(file_path, encoding: 'UTF-8'))
182
+
183
+ orphaned_lines.each do |line_num|
184
+ parent = resource_index.base_key_at(line_num)
185
+ next unless parent
186
+
187
+ keys << parent
188
+ locations[parent] << line_num if locations
189
+ end
190
+ end
191
+
192
+ def add_comment_only_locations(locations, added_lines, file_path, format:, content: nil)
193
+ return if content.nil? && !File.file?(file_path)
194
+
195
+ comment_index = TranslationCommentIndex.new(file_path, format: format, content: content)
196
+
197
+ added_lines.each do |line_number|
198
+ key = comment_index.key_at(line_number)
199
+ next unless key
200
+ next if locations[key].any?
201
+
202
+ locations[key] << yield(line_number)
203
+ end
204
+ end
205
+
206
+ def merge_removed_xml_locations!(locations, diff_output, file_path, base_content:, head_content:)
207
+ return unless base_content && head_content
208
+
209
+ base_comments = TranslationCommentIndex.new(format: :xml, content: base_content)
210
+ head_comments = TranslationCommentIndex.new(format: :xml, content: head_content)
211
+ base_resources = AndroidResource.index(base_content)
212
+ head_resources = AndroidResource.index(head_content)
213
+
214
+ each_changed_diff_line(diff_output) do |content, old_line, _new_line, side|
215
+ next unless side == :left
216
+ next if content.strip.empty?
217
+
218
+ key = base_comments.key_at(old_line) || base_resources.base_key_at(old_line)
219
+ next unless key
220
+
221
+ fallback_line = head_comments.line_for_key(key) || resource_start_line(head_resources, key)
222
+ (locations[key] ||= []) << changed_location(
223
+ file_path,
224
+ old_line,
225
+ side: :left,
226
+ fallback_line: fallback_line
227
+ )
228
+ end
229
+ end
230
+
231
+ def resource_start_line(index, key)
232
+ index.resource_spans.find { |span| span.base_key == key }&.start_line
233
+ end
234
+ end
235
+ end