i18n-context-generator 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +163 -26
  3. data/lib/i18n_context_generator/android_resource.rb +209 -0
  4. data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
  5. data/lib/i18n_context_generator/cache.rb +40 -8
  6. data/lib/i18n_context_generator/changed_location.rb +55 -0
  7. data/lib/i18n_context_generator/cli.rb +163 -76
  8. data/lib/i18n_context_generator/config/cli_values.rb +33 -0
  9. data/lib/i18n_context_generator/config/defaults.rb +35 -0
  10. data/lib/i18n_context_generator/config/schema.rb +225 -0
  11. data/lib/i18n_context_generator/config/serialization.rb +64 -0
  12. data/lib/i18n_context_generator/config/validation.rb +249 -0
  13. data/lib/i18n_context_generator/config.rb +293 -102
  14. data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
  15. data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
  16. data/lib/i18n_context_generator/context_extractor/run_logging.rb +45 -6
  17. data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
  18. data/lib/i18n_context_generator/context_extractor/source_filters.rb +51 -5
  19. data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
  20. data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
  21. data/lib/i18n_context_generator/context_extractor.rb +207 -216
  22. data/lib/i18n_context_generator/file_classifier.rb +72 -0
  23. data/lib/i18n_context_generator/generated_comment.rb +32 -0
  24. data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
  25. data/lib/i18n_context_generator/git_diff.rb +224 -95
  26. data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
  27. data/lib/i18n_context_generator/llm/client.rb +294 -92
  28. data/lib/i18n_context_generator/llm/openai.rb +92 -51
  29. data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
  30. data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
  31. data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
  32. data/lib/i18n_context_generator/localization_syntax.rb +246 -0
  33. data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
  34. data/lib/i18n_context_generator/parsers/base.rb +7 -4
  35. data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
  36. data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
  37. data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
  38. data/lib/i18n_context_generator/path_policy.rb +107 -0
  39. data/lib/i18n_context_generator/platform_validator.rb +24 -50
  40. data/lib/i18n_context_generator/run_metrics.rb +47 -0
  41. data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
  42. data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
  43. data/lib/i18n_context_generator/searcher/source_discovery.rb +109 -64
  44. data/lib/i18n_context_generator/searcher.rb +61 -205
  45. data/lib/i18n_context_generator/supplemental_context.rb +77 -0
  46. data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
  47. data/lib/i18n_context_generator/version.rb +1 -1
  48. data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
  49. data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
  50. data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
  51. data/lib/i18n_context_generator/writers/helpers.rb +7 -29
  52. data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
  53. data/lib/i18n_context_generator/writers/preview.rb +80 -0
  54. data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
  55. data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
  56. data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
  57. data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
  58. data/lib/i18n_context_generator/xcstrings_document.rb +218 -0
  59. data/lib/i18n_context_generator/xml_scanner.rb +38 -0
  60. data/lib/i18n_context_generator.rb +20 -4
  61. metadata +59 -12
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ module LLM
5
+ # Converts application and supplemental inputs into prompt-safe evidence records.
6
+ module PromptEvidence
7
+ private
8
+
9
+ def detect_platform(matches)
10
+ return 'mobile' if matches.empty?
11
+
12
+ platforms = matches.filter_map { |match| FileClassifier.searchable_platform(match.file) }
13
+
14
+ if platforms.include?(:ios)
15
+ 'iOS'
16
+ elsif platforms.include?(:android)
17
+ 'Android'
18
+ else
19
+ 'mobile'
20
+ end
21
+ end
22
+
23
+ def prompt_matches(matches, include_file_paths:, redact_prompts:)
24
+ matches.map do |match|
25
+ location = include_file_paths ? match.file : File.basename(match.file)
26
+ {
27
+ location: sanitized_prompt_value(location, redact: redact_prompts),
28
+ line: match.line,
29
+ enclosing_scope: sanitized_prompt_value(match.enclosing_scope, redact: redact_prompts),
30
+ matched_line: sanitized_prompt_value(match.match_line, redact: redact_prompts),
31
+ context: sanitized_prompt_value(match.context, redact: redact_prompts)
32
+ }.compact
33
+ end
34
+ end
35
+
36
+ def prompt_context_sources(sources, redact_prompts:)
37
+ sources.map do |source|
38
+ {
39
+ kind: sanitized_prompt_value(source.kind, redact: redact_prompts),
40
+ name: sanitized_prompt_value(source.name, redact: redact_prompts),
41
+ content: sanitized_prompt_value(source.content, redact: redact_prompts)
42
+ }
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,147 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ module LLM
5
+ # Shared retry and HTTP error handling for remote LLM providers.
6
+ module RequestPolicy
7
+ RequestOutcome = Data.define(:response, :retries) do
8
+ def request_count = retries + 1
9
+ end
10
+
11
+ # Adds run-local attempt counters to a transport exception without
12
+ # changing its public exception class.
13
+ module FailureTelemetry
14
+ attr_reader :request_count, :retries
15
+
16
+ def record_request_failure(retries:)
17
+ @retries = retries
18
+ @request_count = retries + 1
19
+ self
20
+ end
21
+ end
22
+ private_constant :FailureTelemetry
23
+
24
+ MAX_RETRIES = 2
25
+ MAX_RETRY_DELAY = 30.0
26
+ RETRYABLE_STATUS_CODES = [408, 409, 425, 429, 500, 502, 503, 504, 529].freeze
27
+ TRANSIENT_NETWORK_ERRORS = [
28
+ Net::OpenTimeout,
29
+ Net::ReadTimeout,
30
+ Timeout::Error,
31
+ EOFError,
32
+ SocketError,
33
+ Errno::ECONNABORTED,
34
+ Errno::ECONNREFUSED,
35
+ Errno::ECONNRESET,
36
+ Errno::EHOSTUNREACH,
37
+ Errno::ENETUNREACH,
38
+ Errno::ETIMEDOUT,
39
+ OpenSSL::SSL::SSLError
40
+ ].freeze
41
+
42
+ protected
43
+
44
+ def request_with_retries(uri:)
45
+ retries = 0
46
+
47
+ loop do
48
+ response = yield
49
+ return RequestOutcome.new(response: response, retries: retries) unless retryable_response?(response) && retries < MAX_RETRIES
50
+
51
+ retries += 1
52
+ delay = retry_delay(response, retries)
53
+ reset_http_session(uri)
54
+ sleep(delay)
55
+ rescue *TRANSIENT_NETWORK_ERRORS => e
56
+ if retries >= MAX_RETRIES
57
+ e.extend(FailureTelemetry).record_request_failure(retries: retries)
58
+ raise
59
+ end
60
+
61
+ retries += 1
62
+ reset_http_session(uri)
63
+ sleep(retry_delay(nil, retries))
64
+ retry
65
+ end
66
+ end
67
+
68
+ def retryable_response?(response)
69
+ status = response.code.to_i
70
+ RETRYABLE_STATUS_CODES.include?(status)
71
+ end
72
+
73
+ def retry_delay(response, retry_number)
74
+ retry_after = parse_retry_after(response&.[]('retry-after'))
75
+ return [retry_after, MAX_RETRY_DELAY].min if retry_after
76
+
77
+ base = 0.5 * (2**(retry_number - 1))
78
+ [base + (rand * 0.25), MAX_RETRY_DELAY].min
79
+ end
80
+
81
+ def parse_retry_after(value)
82
+ return nil if value.nil? || value.to_s.strip.empty?
83
+
84
+ numeric = Float(value, exception: false)
85
+ return [numeric, 0].max if numeric&.finite?
86
+
87
+ [Time.httpdate(value) - Time.now, 0].max
88
+ rescue ArgumentError
89
+ nil
90
+ end
91
+
92
+ def http_error_result(response, retries: 0)
93
+ telemetry = request_telemetry(retries: retries)
94
+ case response.code.to_i
95
+ when 401
96
+ ContextResult.new(
97
+ description: 'Authentication failed',
98
+ error: 'Provider rejected the API credentials',
99
+ **telemetry
100
+ )
101
+ when 429
102
+ ContextResult.new(
103
+ description: 'Rate limited',
104
+ error: 'Rate limit exceeded - try reducing concurrency',
105
+ **telemetry
106
+ )
107
+ else
108
+ ContextResult.new(description: 'API error', error: provider_error_message(response), **telemetry)
109
+ end
110
+ end
111
+
112
+ def provider_error_message(response)
113
+ body = JSON.parse(response.body)
114
+ return "HTTP #{response.code}" unless body.is_a?(Hash)
115
+
116
+ message = body.dig('error', 'message') || body['message']
117
+ return "HTTP #{response.code}" unless message.is_a?(String) && !message.empty?
118
+
119
+ message.gsub(/[\u0000-\u001F\u007F]/, ' ')[0, 500]
120
+ rescue JSON::ParserError, TypeError
121
+ "HTTP #{response.code}"
122
+ end
123
+
124
+ def request_telemetry(retries:)
125
+ { request_count: retries + 1, retries: retries }
126
+ end
127
+
128
+ def failure_request_telemetry(error, outcome:)
129
+ return request_telemetry(retries: outcome.retries) if outcome
130
+ return { request_count: error.request_count, retries: error.retries } if
131
+ error.respond_to?(:request_count) && error.respond_to?(:retries)
132
+
133
+ {}
134
+ end
135
+
136
+ def reset_http_session(uri)
137
+ sessions = Thread.current.thread_variable_get(http_sessions_key)
138
+ return unless sessions
139
+
140
+ http = sessions.delete([uri.scheme, uri.host, uri.port])
141
+ http.finish if http&.started?
142
+ rescue IOError, SystemCallError
143
+ nil
144
+ end
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,246 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'android_resource'
4
+ require_relative 'apple_string_literal'
5
+
6
+ module I18nContextGenerator
7
+ # Registry of localization call/resource syntaxes shared by source search,
8
+ # source-first discovery, and Swift comment write-back.
9
+ class LocalizationSyntax
10
+ DEFAULT_SWIFT_FUNCTIONS = %w[
11
+ NSLocalizedString
12
+ String(localized:
13
+ Text(
14
+ LocalizedStringResource(
15
+ ].freeze
16
+
17
+ IOS_STATIC_SEARCH_BUILDERS = [
18
+ ->(key) { "LocalizedStringKey\\s*\\(\\s*\"#{key}\"" },
19
+ ->(key) { "LocalizedStringKey\\s*=\\s*\"#{key}\"" },
20
+ ->(key) { ":\\s*LocalizedStringResource\\s*=\\s*\"#{key}\"" },
21
+ ->(key) { "\"#{key}\"\\.localized" }
22
+ ].freeze
23
+ SWIFT_STRING_BODY_PATTERN = AppleStringLiteral::BODY_PATTERN
24
+ private_constant :SWIFT_STRING_BODY_PATTERN
25
+
26
+ IOS_STATIC_SINGLE_LINE_DISCOVERY_PATTERNS = [
27
+ /LocalizedStringKey\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\s*\)/,
28
+ /:\s*LocalizedStringKey\s*=\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/,
29
+ /:\s*LocalizedStringResource\s*=\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/,
30
+ /"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\.localized\b/
31
+ ].freeze
32
+
33
+ LOCALIZED_RESOURCE_SINGLE_LINE_DISCOVERY_PATTERNS = [
34
+ /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"[^\n]*?\bdefaultValue:\s*"(?<text>#{SWIFT_STRING_BODY_PATTERN})"/,
35
+ /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/
36
+ ].freeze
37
+
38
+ IOS_STATIC_MULTILINE_DISCOVERY_PATTERNS = [
39
+ /Text\s*\(\s*LocalizedStringKey\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"\s*\)[\s\S]*?\)/
40
+ ].freeze
41
+
42
+ LOCALIZED_RESOURCE_MULTILINE_DISCOVERY_PATTERNS = [
43
+ /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"[\s\S]*?\bdefaultValue:\s*"(?<text>#{SWIFT_STRING_BODY_PATTERN})"/,
44
+ /LocalizedStringResource\s*\(\s*"(?<key>#{SWIFT_STRING_BODY_PATTERN})"/
45
+ ].freeze
46
+
47
+ OPTIONAL_ARGUMENT_LABEL_PATTERN = '(?:[A-Za-z_]\w*\s*:\s*)?'
48
+ private_constant :OPTIONAL_ARGUMENT_LABEL_PATTERN
49
+
50
+ ANDROID_DISCOVERY_PATTERNS = {
51
+ string: %r{R\.string\.(\w+)\b|@string/([\w.]+)\b|[(\s,=]string\.(\w+)\b}x,
52
+ plural: %r{R\.plurals\.(\w+)\b|@plurals/([\w.]+)\b|[(\s,=]plurals\.(\w+)\b}x,
53
+ array: %r{R\.array\.(\w+)\b|@array/([\w.]+)\b|[(\s,=]array\.(\w+)\b}x
54
+ }.freeze
55
+
56
+ def self.functions_with_defaults(functions)
57
+ configured = Array(functions).map(&:strip).reject(&:empty?)
58
+ (DEFAULT_SWIFT_FUNCTIONS + configured).uniq { |function| function.sub(/\(\s*\z/, '') }.freeze
59
+ end
60
+
61
+ def self.swift_string_content_pattern(value)
62
+ encoded = value.each_char.map do |character|
63
+ case character
64
+ when '\\' then '\\\\'
65
+ when '"' then '\\"'
66
+ when "\n" then '\n'
67
+ when "\r" then '\r'
68
+ when "\t" then '\t'
69
+ else character
70
+ end
71
+ end.join
72
+ Regexp.escape(encoded)
73
+ end
74
+
75
+ def initialize(swift_functions: nil)
76
+ @swift_functions = self.class.functions_with_defaults(swift_functions)
77
+ end
78
+
79
+ attr_reader :swift_functions
80
+
81
+ def search_patterns(key, platform:, resource_type: nil)
82
+ strings = case platform.to_sym
83
+ when :ios then ios_search_patterns(key)
84
+ when :android then android_search_patterns(key, resource_type: resource_type)
85
+ else
86
+ ios_search_patterns(key) + android_search_patterns(key, resource_type: resource_type) +
87
+ [Regexp.escape(key)]
88
+ end
89
+ strings.map { |pattern| Regexp.new(pattern) }
90
+ end
91
+
92
+ def ios_search_patterns(key)
93
+ escaped_key = self.class.swift_string_content_pattern(key)
94
+ function_patterns = swift_function_key_patterns(escaped_key)
95
+ function_patterns + IOS_STATIC_SEARCH_BUILDERS.map { |builder| builder.call(escaped_key) }
96
+ end
97
+
98
+ def ios_multiline_search_patterns(key)
99
+ escaped_key = self.class.swift_string_content_pattern(key)
100
+ swift_function_key_patterns(escaped_key).map { |pattern| Regexp.new(pattern) }
101
+ end
102
+
103
+ def android_search_patterns(key, resource_type: nil)
104
+ base_key = AndroidResource.base_key(key)
105
+ escaped_base = Regexp.escape(base_key)
106
+
107
+ case AndroidResource.type_for(key, explicit: resource_type)
108
+ when :plural then android_plural_patterns(escaped_base)
109
+ when :array then android_array_patterns(escaped_base)
110
+ else android_string_patterns(Regexp.escape(key))
111
+ end
112
+ end
113
+
114
+ def ios_single_line_discovery_patterns
115
+ @ios_single_line_discovery_patterns ||=
116
+ (LOCALIZED_RESOURCE_SINGLE_LINE_DISCOVERY_PATTERNS +
117
+ function_discovery_patterns(multiline: false) +
118
+ IOS_STATIC_SINGLE_LINE_DISCOVERY_PATTERNS).freeze
119
+ end
120
+
121
+ def ios_multiline_discovery_patterns
122
+ @ios_multiline_discovery_patterns ||=
123
+ (LOCALIZED_RESOURCE_MULTILINE_DISCOVERY_PATTERNS +
124
+ function_discovery_patterns(multiline: true) +
125
+ IOS_STATIC_MULTILINE_DISCOVERY_PATTERNS).freeze
126
+ end
127
+
128
+ def ios_call_start_patterns
129
+ @ios_call_start_patterns ||=
130
+ @swift_functions.map { |function| Regexp.new(swift_function_opener(function)) }.freeze
131
+ end
132
+
133
+ def ios_wrapper_definition_pattern
134
+ @ios_wrapper_definition_pattern ||= begin
135
+ functions = (@swift_functions.map { |function| swift_call_prefix(function) } +
136
+ ['LocalizedStringKey\\s*\\(']).join('|')
137
+ constructor = "\\s*=\\s*(?:#{functions})"
138
+ typed_resource = '\s*:\s*LocalizedStringResource\s*=\s*"'
139
+ /\bstatic\s+(?:let|var)\s+(?<member_name>\w+)(?:#{constructor}|#{typed_resource})/
140
+ end
141
+ end
142
+
143
+ def android_discovery_patterns
144
+ ANDROID_DISCOVERY_PATTERNS
145
+ end
146
+
147
+ def swift_writer_patterns(key)
148
+ escaped_key = self.class.swift_string_content_pattern(key)
149
+ comment = 'comment:\\s*"(?:\\\\.|[^"\\\\])*"'
150
+
151
+ patterns = @swift_functions.map do |function|
152
+ Regexp.new(
153
+ "#{swift_key_argument_prefix(function)}\"#{escaped_key}\"[^)]*#{comment}[^)]*\\)",
154
+ Regexp::MULTILINE
155
+ )
156
+ end
157
+ patterns << nested_text_writer_pattern(escaped_key, comment)
158
+ patterns
159
+ end
160
+
161
+ private
162
+
163
+ def function_discovery_patterns(multiline:)
164
+ tail = multiline ? '[\\s\\S]*?' : '[^\\n]*?'
165
+ @swift_functions.flat_map do |function|
166
+ prefix = swift_key_argument_prefix(function)
167
+ [
168
+ Regexp.new("#{prefix}\\s*@?\"(?<key>#{SWIFT_STRING_BODY_PATTERN})\"#{tail}comment:\\s*\"(?<comment>#{SWIFT_STRING_BODY_PATTERN})\""),
169
+ Regexp.new("#{prefix}\\s*@?\"(?<key>#{SWIFT_STRING_BODY_PATTERN})\"")
170
+ ]
171
+ end
172
+ end
173
+
174
+ def swift_function_key_patterns(escaped_key)
175
+ @swift_functions.map do |function|
176
+ "#{swift_key_argument_prefix(function)}@?\"#{escaped_key}\""
177
+ end
178
+ end
179
+
180
+ def swift_key_argument_prefix(function)
181
+ prefix = swift_call_prefix(function)
182
+ return "#{prefix}\\s*" if explicit_argument_fragment?(function) || DEFAULT_SWIFT_FUNCTIONS.include?(function)
183
+
184
+ "#{prefix}\\s*#{OPTIONAL_ARGUMENT_LABEL_PATTERN}"
185
+ end
186
+
187
+ def swift_call_prefix(function)
188
+ name, arguments = function.split('(', 2)
189
+ prefix = "#{Regexp.escape(name.strip)}\\s*\\("
190
+ return prefix unless arguments
191
+
192
+ arguments = arguments.strip
193
+ arguments.empty? ? prefix : "#{prefix}\\s*#{Regexp.escape(arguments)}"
194
+ end
195
+
196
+ def swift_function_opener(function)
197
+ name = function.split('(', 2).first
198
+ "#{Regexp.escape(name.strip)}\\s*\\("
199
+ end
200
+
201
+ def explicit_argument_fragment?(function)
202
+ _name, arguments = function.split('(', 2)
203
+ arguments && !arguments.strip.empty?
204
+ end
205
+
206
+ def nested_text_writer_pattern(escaped_key, comment)
207
+ Regexp.new(
208
+ "Text\\s*\\(\\s*LocalizedStringKey\\s*\\(\\s*\"#{escaped_key}\"\\s*\\)" \
209
+ "[^)]*#{comment}[^)]*\\)",
210
+ Regexp::MULTILINE
211
+ )
212
+ end
213
+
214
+ def android_plural_patterns(key)
215
+ [
216
+ "R\\.plurals\\.#{key}\\b", "@plurals/#{key}\\b",
217
+ "getQuantityString\\s*\\(\\s*R\\.plurals\\.#{key}",
218
+ "\\.getQuantityString\\s*\\(\\s*R\\.plurals\\.#{key}",
219
+ "pluralStringResource\\s*\\(\\s*R\\.plurals\\.#{key}",
220
+ "[\\(\\s,=]plurals\\.#{key}\\b"
221
+ ]
222
+ end
223
+
224
+ def android_array_patterns(key)
225
+ [
226
+ "R\\.array\\.#{key}\\b", "@array/#{key}\\b",
227
+ "getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
228
+ "\\.getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
229
+ "resources\\.getStringArray\\s*\\(\\s*R\\.array\\.#{key}",
230
+ "[\\(\\s,=]array\\.#{key}\\b"
231
+ ]
232
+ end
233
+
234
+ def android_string_patterns(key)
235
+ [
236
+ "R\\.string\\.#{key}\\b", "@string/#{key}\\b",
237
+ "getString\\s*\\(\\s*R\\.string\\.#{key}",
238
+ "\\.getString\\s*\\(\\s*R\\.string\\.#{key}",
239
+ "stringResource\\s*\\(\\s*R\\.string\\.#{key}",
240
+ "[\\(\\s,=]string\\.#{key}\\b",
241
+ "getString\\s*\\(\\s*string\\.#{key}",
242
+ "stringResource\\s*\\(\\s*string\\.#{key}"
243
+ ]
244
+ end
245
+ end
246
+ end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'rexml/document'
4
+ require_relative '../android_resource'
4
5
 
5
6
  module I18nContextGenerator
6
7
  module Parsers
@@ -11,6 +12,7 @@ module I18nContextGenerator
11
12
  def parse(path)
12
13
  content = File.read(path, encoding: 'UTF-8')
13
14
  doc = REXML::Document.new(content)
15
+ resource_index = AndroidResource.index(content)
14
16
  entries = []
15
17
 
16
18
  doc.elements.each('resources/string') do |element|
@@ -22,11 +24,12 @@ module I18nContextGenerator
22
24
  # Look for preceding comment
23
25
  comment = find_preceding_comment(element)
24
26
 
25
- entries << TranslationEntry.new(
27
+ entries << build_entry(
26
28
  key: key,
27
29
  text: unescape_android_string(text),
28
30
  source_file: path,
29
- metadata: { comment: comment }
31
+ metadata: { comment: comment, resource_type: :string },
32
+ resource_index: resource_index
30
33
  )
31
34
  end
32
35
 
@@ -35,12 +38,22 @@ module I18nContextGenerator
35
38
  next unless translatable?(array_element)
36
39
 
37
40
  array_name = array_element.attributes['name']
41
+ parent_comment = find_preceding_comment(array_element)
38
42
  array_element.elements.to_a('item').each_with_index do |item, index|
39
- entries << TranslationEntry.new(
40
- key: "#{array_name}[#{index}]",
43
+ key = AndroidResource.composite_key(array_name, type: :array, index: index)
44
+ item_comment = find_preceding_comment(item)
45
+ entries << build_entry(
46
+ key: key,
41
47
  text: unescape_android_string(inner_text(item)),
42
48
  source_file: path,
43
- metadata: { array: array_name, index: index }
49
+ metadata: {
50
+ array: array_name,
51
+ index: index,
52
+ resource_type: :array,
53
+ parent_comment: parent_comment,
54
+ comment: combined_comment(parent_comment, item_comment)
55
+ }.compact,
56
+ resource_index: resource_index
44
57
  )
45
58
  end
46
59
  end
@@ -50,22 +63,44 @@ module I18nContextGenerator
50
63
  next unless translatable?(plural_element)
51
64
 
52
65
  plural_name = plural_element.attributes['name']
66
+ parent_comment = find_preceding_comment(plural_element)
53
67
  plural_element.elements.each('item') do |item|
54
68
  quantity = item.attributes['quantity']
55
- entries << TranslationEntry.new(
56
- key: "#{plural_name}:#{quantity}",
69
+ key = AndroidResource.composite_key(plural_name, type: :plural, quantity: quantity)
70
+ item_comment = find_preceding_comment(item)
71
+ entries << build_entry(
72
+ key: key,
57
73
  text: unescape_android_string(inner_text(item)),
58
74
  source_file: path,
59
- metadata: { plural: plural_name, quantity: quantity }
75
+ metadata: {
76
+ plural: plural_name,
77
+ quantity: quantity,
78
+ resource_type: :plural,
79
+ parent_comment: parent_comment,
80
+ comment: combined_comment(parent_comment, item_comment)
81
+ }.compact,
82
+ resource_index: resource_index
60
83
  )
61
84
  end
62
85
  end
63
86
 
64
87
  entries
88
+ rescue REXML::ParseException => e
89
+ raise Error, "Failed to parse Android XML translation file #{path}: #{e.message.lines.first&.strip}"
65
90
  end
66
91
 
67
92
  private
68
93
 
94
+ def build_entry(key:, text:, source_file:, metadata:, resource_index:)
95
+ span = resource_index.span_for(key)
96
+ if span
97
+ metadata = metadata.merge(
98
+ line_span: span.line_span
99
+ )
100
+ end
101
+ TranslationEntry.new(key: key, text: text, source_file: source_file, metadata: metadata)
102
+ end
103
+
69
104
  # Get the full inner content of an element, including inline markup like
70
105
  # <b>, <i>, <u>, <xliff:g>. REXML::Element#text only returns the first
71
106
  # text node, losing everything after a nested element.
@@ -91,6 +126,10 @@ module I18nContextGenerator
91
126
  nil
92
127
  end
93
128
 
129
+ def combined_comment(parent_comment, item_comment)
130
+ [parent_comment, item_comment].compact.uniq.join("\n").then { |comment| comment unless comment.empty? }
131
+ end
132
+
94
133
  def translatable?(element)
95
134
  element.attributes['translatable']&.downcase != 'false'
96
135
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../file_classifier'
4
+
3
5
  module I18nContextGenerator
4
6
  module Parsers
5
7
  # Represents a single translation entry
@@ -11,20 +13,21 @@ module I18nContextGenerator
11
13
 
12
14
  # Base class for translation file parsers
13
15
  class Base
14
- def self.for(path)
15
- basename = File.basename(path).downcase
16
+ def self.for(path, locale: nil)
16
17
  ext = File.extname(path).downcase
17
18
 
18
19
  case ext
19
20
  when '.json'
20
21
  JsonParser.new
21
22
  when '.yml', '.yaml'
22
- YamlParser.new
23
+ YamlParser.new(locale: locale)
23
24
  when '.strings'
24
25
  StringsParser.new
26
+ when '.xcstrings'
27
+ XcstringsParser.new
25
28
  when '.xml'
26
29
  # Check if it's an Android strings.xml
27
- raise Error, "Unsupported XML format: #{path} (only Android strings.xml is supported)" unless basename == 'strings.xml' || path.include?('/res/values')
30
+ raise Error, "Unsupported XML format: #{path} (only Android strings.xml is supported)" unless FileClassifier.android_translation_file?(path)
28
31
 
29
32
  AndroidXmlParser.new
30
33
 
@@ -6,6 +6,8 @@ module I18nContextGenerator
6
6
  class JsonParser < Base
7
7
  def parse(path)
8
8
  data = Oj.load_file(path)
9
+ raise Error, "Invalid JSON translation file #{path}: root must be an object" unless data.is_a?(Hash)
10
+
9
11
  flatten_keys(data).filter_map do |key, text|
10
12
  next if text.nil? || text.to_s.strip.empty?
11
13
 
@@ -15,6 +17,8 @@ module I18nContextGenerator
15
17
  source_file: path
16
18
  )
17
19
  end
20
+ rescue Oj::ParseError => e
21
+ raise Error, "Failed to parse JSON translation file #{path}: #{e.message}"
18
22
  end
19
23
  end
20
24
  end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ module Parsers
5
+ # Parser for Apple string catalogs. Context generation is keyed by the
6
+ # catalog entry rather than by individual target-language localizations.
7
+ class XcstringsParser < Base
8
+ def parse(path)
9
+ catalog = load_catalog(path)
10
+ source_language = catalog['sourceLanguage']
11
+
12
+ catalog.fetch('strings').filter_map do |key, entry|
13
+ next if key.empty?
14
+ next if entry['shouldTranslate'] == false
15
+
16
+ TranslationEntry.new(
17
+ key: key,
18
+ text: source_text(key, entry, source_language),
19
+ source_file: path,
20
+ metadata: {
21
+ comment: entry['comment'],
22
+ source_language: source_language,
23
+ resource_type: :string
24
+ }.compact
25
+ )
26
+ end
27
+ rescue KeyError => e
28
+ raise Error, "Failed to parse Apple string catalog #{path}: missing #{e.key.inspect}"
29
+ rescue Oj::ParseError, TypeError => e
30
+ raise Error, "Failed to parse Apple string catalog #{path}: #{e.message}"
31
+ end
32
+
33
+ def self.validate_catalog!(catalog, path:)
34
+ raise TypeError, 'root must be a mapping' unless catalog.is_a?(Hash)
35
+ raise TypeError, 'sourceLanguage must be a non-empty string' unless nonempty_string?(catalog['sourceLanguage'])
36
+ raise TypeError, 'strings must be a mapping' unless catalog['strings'].is_a?(Hash)
37
+
38
+ valid_entries = catalog['strings'].all? { |key, entry| key.is_a?(String) && entry.is_a?(Hash) }
39
+ raise TypeError, 'strings must map string keys to entries' unless valid_entries
40
+
41
+ catalog
42
+ rescue TypeError => e
43
+ raise Error, "Failed to parse Apple string catalog #{path}: #{e.message}"
44
+ end
45
+
46
+ def self.nonempty_string?(value)
47
+ value.is_a?(String) && !value.empty?
48
+ end
49
+ private_class_method :nonempty_string?
50
+
51
+ private
52
+
53
+ def load_catalog(path)
54
+ catalog = Oj.load_file(path, mode: :strict)
55
+ self.class.validate_catalog!(catalog, path: path)
56
+ end
57
+
58
+ def source_text(key, entry, source_language)
59
+ localization = entry.dig('localizations', source_language)
60
+ values = string_unit_values(localization)
61
+ values.empty? ? key : values.join(' | ')
62
+ end
63
+
64
+ def string_unit_values(value)
65
+ case value
66
+ when Hash
67
+ direct_value = value.dig('stringUnit', 'value')
68
+ return [direct_value] if direct_value.is_a?(String)
69
+
70
+ value.values.flat_map { |child| string_unit_values(child) }
71
+ when Array
72
+ value.flat_map { |child| string_unit_values(child) }
73
+ else
74
+ []
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end