i18n-context-generator 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +163 -26
  3. data/lib/i18n_context_generator/android_resource.rb +209 -0
  4. data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
  5. data/lib/i18n_context_generator/cache.rb +40 -8
  6. data/lib/i18n_context_generator/changed_location.rb +55 -0
  7. data/lib/i18n_context_generator/cli.rb +165 -73
  8. data/lib/i18n_context_generator/config/cli_values.rb +33 -0
  9. data/lib/i18n_context_generator/config/defaults.rb +35 -0
  10. data/lib/i18n_context_generator/config/schema.rb +225 -0
  11. data/lib/i18n_context_generator/config/serialization.rb +64 -0
  12. data/lib/i18n_context_generator/config/validation.rb +249 -0
  13. data/lib/i18n_context_generator/config.rb +315 -105
  14. data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
  15. data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
  16. data/lib/i18n_context_generator/context_extractor/run_logging.rb +65 -0
  17. data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
  18. data/lib/i18n_context_generator/context_extractor/source_filters.rb +104 -0
  19. data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
  20. data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
  21. data/lib/i18n_context_generator/context_extractor.rb +218 -164
  22. data/lib/i18n_context_generator/file_classifier.rb +72 -0
  23. data/lib/i18n_context_generator/generated_comment.rb +32 -0
  24. data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
  25. data/lib/i18n_context_generator/git_diff.rb +280 -88
  26. data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
  27. data/lib/i18n_context_generator/llm/client.rb +294 -92
  28. data/lib/i18n_context_generator/llm/openai.rb +92 -51
  29. data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
  30. data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
  31. data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
  32. data/lib/i18n_context_generator/localization_syntax.rb +246 -0
  33. data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
  34. data/lib/i18n_context_generator/parsers/base.rb +7 -4
  35. data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
  36. data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
  37. data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
  38. data/lib/i18n_context_generator/path_policy.rb +107 -0
  39. data/lib/i18n_context_generator/platform_validator.rb +24 -50
  40. data/lib/i18n_context_generator/run_metrics.rb +47 -0
  41. data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
  42. data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
  43. data/lib/i18n_context_generator/searcher/source_discovery.rb +232 -0
  44. data/lib/i18n_context_generator/searcher.rb +69 -201
  45. data/lib/i18n_context_generator/supplemental_context.rb +77 -0
  46. data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
  47. data/lib/i18n_context_generator/version.rb +1 -1
  48. data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
  49. data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
  50. data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
  51. data/lib/i18n_context_generator/writers/helpers.rb +7 -29
  52. data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
  53. data/lib/i18n_context_generator/writers/preview.rb +80 -0
  54. data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
  55. data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
  56. data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
  57. data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
  58. data/lib/i18n_context_generator/xcstrings_document.rb +218 -0
  59. data/lib/i18n_context_generator/xml_scanner.rb +38 -0
  60. data/lib/i18n_context_generator.rb +20 -4
  61. metadata +62 -12
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bfe865e8057b9ad9e5a239beca369736c430b118ba138ebbb2696be2d1c64af3
4
- data.tar.gz: 7669dbee58f5030654eb40115db899d650b3bcfa379738c48242dbac2a59e0dc
3
+ metadata.gz: 6588c09e273cb74c40dba8cc26798e46b82b5d47e5c840f6e4609843bb1f5ed8
4
+ data.tar.gz: 9b3e309fd4030a99e065a2c7693240f09d205f9d95323a9744ecb3eaa1b239b1
5
5
  SHA512:
6
- metadata.gz: 6c544f04ecabc25b2d6e2e196f1aeb7b0d0dd0bfa17be8ad57bef46e02d4e4741ffd2c6187d7e36acbccdedd9826a963d3ef4fb70825a988b55b11b76c030465
7
- data.tar.gz: 8878af77894c57dbdc169bed6ae982889698b3121dca6bbc59c0af9099c520e272145ef9d1324d184dba9af5cf218d7cb7b9965b4369b8ac3a3c55d0a4e461c1
6
+ metadata.gz: 2a7b9df974f135375341c7c1a5f73e40575f10c92d11a72ecc976ecef9ae6b25aac32351f1468c79671c870d87c2df576664a96b32ef00128815fc4ff016de1c
7
+ data.tar.gz: 0cc9db25973ffe209dea8cb4197df899310320361f0a8da2e45fa597fc592867bea22cdd5ad5337907c18cbdf824e269955fe79550baedd23f0d919fdef73c00
data/README.md CHANGED
@@ -2,17 +2,18 @@
2
2
 
3
3
  `i18n-context-generator` generates translator-facing context for your existing localization keys. It reads translation files, finds where each key is used in app code, and asks an LLM to explain the string's UI role in plain language.
4
4
 
5
- You can export the results as CSV or JSON, write them back into `.strings` or `strings.xml`, or update Swift `comment:` arguments directly.
5
+ You can export the results as CSV or JSON, write them back into `.strings`, `.xcstrings`, or `strings.xml`, or update Swift `comment:` arguments directly.
6
6
 
7
7
  It is designed for mobile codebases. Each run must target either iOS or Android, not both.
8
8
 
9
9
  ## What It Does
10
10
 
11
- - Parses `.strings`, `strings.xml`, `.json`, and `.yml` or `.yaml` translation files
11
+ - Parses `.strings`, `.xcstrings`, `strings.xml`, `.json`, and `.yml` or `.yaml` translation files
12
12
  - Searches Swift, Objective-C, Kotlin, Java, and Android XML for matching usages
13
- - Uses Anthropic or OpenAI models to infer UI context
13
+ - Uses Anthropic, OpenAI, or an explicitly configured OpenAI-compatible endpoint to infer UI context
14
14
  - Supports diff-based runs, key filters, and key ranges for incremental work
15
- - Redacts likely secrets, URLs, and emails from prompts by default
15
+ - Separates validation, planning, patch preview, and mutation into explicit workflow stages
16
+ - Applies best-effort redaction of likely secrets, URLs, and emails by default
16
17
  - Optionally caches results to avoid repeating identical LLM work
17
18
 
18
19
  ## Installation
@@ -24,13 +25,12 @@ chmod +x exe/i18n-context-generator
24
25
 
25
26
  ## Quick Start
26
27
 
27
- Preview what would be processed:
28
+ Plan what would be processed without calling an LLM:
28
29
 
29
30
  ```bash
30
- bundle exec exe/i18n-context-generator extract \
31
+ bundle exec exe/i18n-context-generator plan \
31
32
  -t ios/MyApp/Resources/Localizable.strings \
32
- -s ios/MyApp \
33
- --dry-run
33
+ -s ios/MyApp
34
34
  ```
35
35
 
36
36
  For a real run, set a provider API key:
@@ -77,7 +77,7 @@ bundle exec exe/i18n-context-generator extract \
77
77
  --write-back-to-code
78
78
  ```
79
79
 
80
- Use `--output`, `--write-back`, or `--write-back-to-code` depending on where you want results to go. If you have both iOS and Android code in the same repository, run the tool separately for each platform.
80
+ Use `--output`, `--stdout`, `--write-back`, or `--write-back-to-code` depending on where you want results to go. `extract` uses the configured `workflow.stage`, which defaults to `apply`; the named workflow commands always select their corresponding stage explicitly. If you have both iOS and Android code in the same repository, run the tool separately for each platform.
81
81
 
82
82
  ## Configuration
83
83
 
@@ -91,35 +91,56 @@ bundle exec exe/i18n-context-generator extract --config .i18n-context-generator.
91
91
  Example `.i18n-context-generator.yml`:
92
92
 
93
93
  ```yaml
94
+ schema_version: 1
95
+
94
96
  translations:
95
97
  - path: ios/MyApp/Resources/Localizable.strings
96
98
 
97
99
  source:
98
100
  paths:
99
101
  - ios/MyApp
102
+ # Optional additions to the built-in dependency, build, and test ignores
100
103
  ignore:
101
- - "**/Pods/**"
102
- - "**/build/**"
103
- - "**/*Tests*"
104
+ - "**/*.generated.*"
105
+
106
+ context:
107
+ # Free-form text or Markdown included in full for every selected key.
108
+ files:
109
+ - GLOSSARY.md
110
+ - docs/localization-style.md
104
111
 
105
112
  llm:
106
113
  provider: anthropic
107
- model: claude-sonnet-4-6
114
+ # Optional; each provider has its own default model.
115
+ # model: claude-sonnet-4-6
116
+ # openai_compatible requires an explicit model and endpoint.
117
+ # endpoint: http://127.0.0.1:11434/v1/responses
108
118
 
109
119
  processing:
110
120
  concurrency: 5
111
121
  context_lines: 15
112
122
  max_matches_per_key: 3
123
+ max_prompt_chars: 50000
124
+
125
+ cache:
126
+ enabled: false
127
+ directory: .i18n-context-generator-cache
128
+
129
+ workflow:
130
+ stage: apply
113
131
 
114
132
  output:
115
133
  format: csv
116
134
  path: translation-context.csv
135
+ stdout: false
117
136
  write_back: false
118
137
  write_back_to_code: false
119
138
  context_prefix: "Context: "
120
139
  context_mode: replace
121
140
 
122
141
  swift:
142
+ # These syntaxes are used by extraction and Swift comment write-back.
143
+ # Custom entries extend the built-in defaults shown below.
123
144
  functions:
124
145
  - NSLocalizedString
125
146
  - "String(localized:"
@@ -128,48 +149,96 @@ swift:
128
149
  privacy:
129
150
  include_file_paths: false
130
151
  include_translation_comments: true
152
+ # Best-effort only; source snippets are still sent to the remote provider.
131
153
  redact_prompts: true
132
154
  ```
133
155
 
134
156
  Use a separate config for Android instead of mixing iOS and Android paths in the same run.
157
+ Client `source.ignore` entries extend the built-in ignore list; they do not replace it.
158
+ Configured `swift.functions` extend the built-in defaults and are used consistently for usage search, source-first discovery, and `--write-back-to-code`. A custom function may be written as either `MyLocalizedString` or `MyLocalizedString(`; its first string argument, with or without a label such as `key:`, is treated as the translation key.
159
+ Unversioned configuration files use schema version 1 compatibility mode: unknown options produce warnings and remain ignored as they were before versioning. New files should declare `schema_version: 1`; explicit versioning enables strict unknown-option validation, and future versions fail clearly. Run `i18n-context-generator config validate [PATH]` to validate a file or add `--print-config` to an extraction command to inspect the fully resolved, secret-free configuration.
135
160
 
136
161
  ## CLI Reference
137
162
 
138
163
  ### Inputs and Output
139
164
 
140
- - `-t`, `--translations FILES`: translation files to process, comma-separated
141
- - `-s`, `--source DIRS`: source files or directories to search, comma-separated
165
+ - `-t`, `--translation FILE`: translation file to process; repeat for multiple files
166
+ - `-s`, `--source PATH`: source file or directory to search; repeat for multiple paths
167
+ - `--context-file PATH`: free-form supplemental context file; repeat for multiple files
142
168
  - `-c`, `--config PATH`: load options from `.i18n-context-generator.yml`
143
169
  - `-o`, `--output PATH`: write results to a file
170
+ - `--stdout`: write structured CSV or JSON to standard output
144
171
  - `-f`, `--format csv|json`: output format, default `csv`
172
+ - `--print-config`: print the resolved configuration and exit
145
173
 
146
174
  ### LLM Settings
147
175
 
148
- - `-p`, `--provider anthropic|openai`: LLM provider, default `anthropic`
176
+ - `-p`, `--provider anthropic|openai|openai_compatible`: LLM provider, default `anthropic`
149
177
  - `-m`, `--model MODEL`: explicit model override
178
+ - `--endpoint URL`: explicit Responses API endpoint for `openai_compatible`
150
179
  - `--concurrency N`: parallel request count, default `5`
180
+ - `--max-prompt-chars N`: hard character limit per LLM prompt, default `50000`
151
181
 
152
182
  ### Filtering and Incremental Runs
153
183
 
154
- - `-k`, `--keys PATTERNS`: wildcard key filter such as `settings.*`
184
+ - `-k`, `--key PATTERN`: wildcard key filter such as `settings.*`; repeat for multiple patterns
155
185
  - `--diff-base REF`: process only keys changed since a Git ref
186
+ - `--diff-head REF`: compare `--diff-base` to this ref, default `HEAD`
156
187
  - `--start-key KEY`: start at a specific key, inclusive
157
188
  - `--end-key KEY`: stop at a specific key, inclusive
158
189
  - `--dry-run`: preview matching keys without calling the LLM
159
190
 
160
191
  ### Write-Back and Prompt Controls
161
192
 
162
- - `--write-back`: update `.strings` or `strings.xml`
193
+ - `--write-back`: update `.strings`, `.xcstrings`, or `strings.xml`
163
194
  - `--write-back-to-code`: update Swift `comment:` arguments
164
195
  - `--context-prefix TEXT`: prefix generated comments, default `Context: `
165
196
  - `--context-mode replace|append`: replace existing comments or append to them
166
197
  - `--cache`: enable on-disk caching
198
+ - `--cache-dir PATH`: cache directory, default `.i18n-context-generator-cache`
167
199
  - `--include-file-paths`: include full source paths in prompts
168
200
  - `--include-translation-comments`: include existing translation comments in prompts, default `true`
169
- - `--redact-prompts`: redact likely secrets and PII before prompts are sent, default `true`
201
+ - `--redact-prompts`: best-effort redact likely secrets and PII before prompts are sent, default `true`
170
202
 
171
203
  Run `bundle exec exe/i18n-context-generator help extract` for the full command reference.
172
204
 
205
+ The legacy `--translations` and `--keys` flags still accept comma-separated lists with a deprecation warning. Repeatable singular flags always preserve literal commas in paths and key patterns.
206
+
207
+ ### Workflow stages
208
+
209
+ - `check`: validate configuration and confirm source usage without calling the LLM
210
+ - `plan`: list selected keys and destinations without calling the LLM
211
+ - `preview-diff`: generate context and print the exact write-back patch to stdout
212
+ without changing translation, source, or structured-output files; diagnostics
213
+ go to stderr
214
+ - `apply`: generate context and write to the configured destinations
215
+ - `extract`: compatibility entry point that uses `workflow.stage` from configuration, defaulting to `apply`
216
+
217
+ `--dry-run` remains available as a compatibility shortcut for non-mutating key selection. Use `preview-diff` when you need to inspect generated write-back content rather than only the selected keys.
218
+
219
+ ### Prompt privacy and size
220
+
221
+ Remote-provider runs send translation text, selected source snippets, configured context files, and programmatic supplemental context off the machine. By default, full paths are reduced to basenames and the tool applies pattern-based redaction to keys, text, comments, paths, scopes, matched lines, surrounding context, and supplemental context. This is a best-effort safeguard, not a guarantee that all private or identifying data will be detected. Review every input you configure before using a remote provider.
222
+
223
+ All context files are treated as opaque, untrusted evidence: they are neither parsed as a glossary nor filtered for relevance, and their complete contents are included in every selected key's request. Named runtime context follows the same contract. Evidence cannot override the system prompt or instruct the model, but prompt-injection defenses are not a substitute for reviewing sensitive inputs.
224
+
225
+ `processing.max_prompt_chars` bounds each user prompt. When evidence exceeds the limit, surrounding code and optional metadata are truncated deterministically, with the translation key, text, and at least one usage receiving priority. Supplemental context is never partially included: if all of it cannot fit alongside the minimum application evidence, extraction fails with guidance to reduce the context or raise the limit. Provider output is validated against the application contract before it can reach an output writer.
226
+
227
+ For a local or third-party Responses API, select `openai_compatible` and configure both `llm.model` and `llm.endpoint`. Plain HTTP is accepted only for loopback endpoints; remote endpoints must use HTTPS. Authentication is isolated to `OPENAI_COMPATIBLE_API_KEY`, so this provider never implicitly forwards `OPENAI_API_KEY` to a custom host.
228
+
229
+ ```yaml
230
+ llm:
231
+ provider: openai_compatible
232
+ model: local-model-name
233
+ endpoint: http://127.0.0.1:11434/v1/responses
234
+ ```
235
+
236
+ ### Cache behavior
237
+
238
+ Caching is opt-in and stores successful results only; provider, resolved model, custom endpoint, prompt controls, source context, source-discovery locations, and supplemental-context content digests are part of each cache identity. Raw context-file and runtime-context contents are not stored in the cache identity. Writes use private temporary files and atomic replacement so concurrent workers cannot leave partial JSON behind.
239
+
240
+ The default directory is `.i18n-context-generator-cache`. Add that directory—or your custom `cache.directory`—to the client repository's `.gitignore`; cached LLM output should not be committed. The generated starter configuration keeps caching disabled by default.
241
+
173
242
  ## Supported Inputs
174
243
 
175
244
  ### Translation Files
@@ -177,15 +246,24 @@ Run `bundle exec exe/i18n-context-generator help extract` for the full command r
177
246
  | Format | Notes |
178
247
  |--------|-------|
179
248
  | `.strings` | Apple strings files |
249
+ | `.xcstrings` | Apple string catalogs; reads the source language, comments, and plural variations |
180
250
  | `strings.xml` | Android string resources, including plurals and arrays |
181
- | `.json` | Nested keys are flattened |
182
- | `.yml`, `.yaml` | Nested keys are flattened |
251
+ | `.json` | Read-only hydration source; nested keys are flattened |
252
+ | `.yml`, `.yaml` | Read-only hydration source; nested keys are flattened and locale roots must be configured explicitly |
253
+
254
+ For Rails-style YAML files with a locale root, name that root in the translation entry. Without `locale:`, the top-level key is retained as part of every translation key; locale-shaped application namespaces are never guessed or removed.
255
+
256
+ ```yaml
257
+ translations:
258
+ - path: config/locales/en.yml
259
+ locale: en
260
+ ```
183
261
 
184
262
  ### Source Search
185
263
 
186
264
  | Platform | Files searched | Typical patterns |
187
265
  |----------|----------------|------------------|
188
- | iOS | `.swift`, `.m`, `.mm`, `.h` | `NSLocalizedString`, `String(localized:)`, `LocalizedStringKey`, `Text`, `.localized` |
266
+ | iOS | `.swift`, `.m`, `.mm`, `.h` | `NSLocalizedString`, `String(localized:)`, `LocalizedStringResource`, `LocalizedStringKey`, `Text`, `.localized` |
189
267
  | Android | `.kt`, `.java`, `.xml` | `R.string.*`, `getString(...)`, `stringResource(...)`, `@string/...`, plurals, arrays |
190
268
 
191
269
  ## Output
@@ -193,12 +271,19 @@ Run `bundle exec exe/i18n-context-generator help extract` for the full command r
193
271
  CSV example:
194
272
 
195
273
  ```csv
196
- key,text,description,ui_element,tone,max_length,locations,error
197
- settings.title,Settings,Navigation bar title for the main settings screen,navigation,neutral,15,ios/SettingsViewController.swift:17,
198
- common.save,Save,Primary action button in forms and edit screens,button,neutral,10,ios/ProfileViewController.swift:31,
199
- error.network,Unable to connect,Error message shown when network requests fail,alert,apologetic,,ios/ProfileViewController.swift:94,
274
+ key,text,description,ui_element,tone,max_length,confidence,ambiguity_reason,locations,status,cache_hit,request_count,input_tokens,output_tokens,retries,error
275
+ settings.title,Settings,Navigation bar title for the main settings screen,navigation,neutral,15,high,,ios/SettingsViewController.swift:17,success,false,1,834,61,0,
200
276
  ```
201
277
 
278
+ JSON and CSV output schema version 1 includes source-file and translation-key
279
+ identity, complete and changed-only evidence locations, side-aware translation
280
+ diff locations, confidence, ambiguity, token counts, retries, and request count.
281
+ This keeps duplicate keys from different translation files distinguishable. The
282
+ run summary reports aggregate requests, cache hits, tokens, retries, and an
283
+ estimated list-price cost when the selected model has known pricing. Cost is
284
+ informational and does not account for provider-specific discounts or billing
285
+ adjustments.
286
+
202
287
  With `--write-back`, generated context is written back into translation files:
203
288
 
204
289
  **Before**
@@ -251,6 +336,58 @@ bundle exec exe/i18n-context-generator extract \
251
336
  --context-prefix ""
252
337
  ```
253
338
 
339
+ `--diff-base` compares against `HEAD` by default. Pass `--diff-head` when a CI system prepares explicit refs:
340
+
341
+ ```bash
342
+ bundle exec exe/i18n-context-generator extract \
343
+ -t ios/Resources/Localizable.strings \
344
+ -s ios \
345
+ --diff-base danger_base \
346
+ --diff-head danger_head \
347
+ --output translation-context.json
348
+ ```
349
+
350
+ Programmatic callers can use the same range and may pass enum-style options as strings or symbols:
351
+
352
+ ```ruby
353
+ config = I18nContextGenerator::Config.new(
354
+ translations: ['ios/Resources/Localizable.strings'],
355
+ source_paths: ['ios'],
356
+ context_files: ['GLOSSARY.md', 'docs/localization-style.md'],
357
+ supplemental_context: {
358
+ 'Pull request title' => 'Clarify Reader renewal labels',
359
+ 'Pull request description' => 'Updates subscription-expiry messaging.'
360
+ },
361
+ discovery_mode: :translations,
362
+ provider: :anthropic,
363
+ diff_base: 'danger_base',
364
+ diff_head: 'danger_head'
365
+ )
366
+
367
+ extractor = I18nContextGenerator::ContextExtractor.new(
368
+ config,
369
+ quiet: true,
370
+ progress: false
371
+ )
372
+ extractor.run
373
+
374
+ extractor.results.each do |result|
375
+ result.status # :success, :no_usage, or :error
376
+ result.actionable? # Safe to present to a user
377
+ result.locations # Every source location used as extraction evidence
378
+ result.changed_locations # Source evidence changed inside the configured range
379
+ result.changed_location_groups # Changed source lines grouped by localization occurrence
380
+ result.translation_key # Owning translation resource (including Android collections)
381
+ # ChangedLocation objects with file, line, side (:left/:right), and an
382
+ # optional right-side fallback_line for review suggestions.
383
+ result.changed_translation_locations
384
+ end
385
+ ```
386
+
387
+ An invalid ref or failed `git diff` raises `I18nContextGenerator::Error`; it is never reported as an unchanged range.
388
+ Programmatic callers may also inject `log_output:`, `structured_output:`, and
389
+ `patch_output:` streams instead of using the process-wide standard streams.
390
+
254
391
  Example GitHub Actions step:
255
392
 
256
393
  ```yaml
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ # Canonical Android string-resource identity and source-span indexing.
5
+ module AndroidResource
6
+ TAG_TYPES = {
7
+ 'string' => :string,
8
+ 'plurals' => :plural,
9
+ 'string-array' => :array
10
+ }.freeze
11
+ TYPE_TAGS = TAG_TYPES.invert.freeze
12
+ RESOURCE_OPEN_PATTERN = /<(?<tag>string-array|plurals|string)\b(?<attributes>[^>]*?)>/m
13
+ ITEM_OPEN_PATTERN = /<item\b(?<attributes>[^>]*?)>/m
14
+
15
+ Span = Data.define(:key, :base_key, :resource_type, :start_line, :end_line, :parent_start_line, :parent_end_line) do
16
+ def line_span
17
+ start_line..end_line
18
+ end
19
+
20
+ def parent_line_span
21
+ parent_start_line..parent_end_line
22
+ end
23
+
24
+ def cover?(line_number)
25
+ line_span.cover?(line_number)
26
+ end
27
+ end
28
+
29
+ # Query object shared by parsers and diff/location consumers.
30
+ class Index
31
+ attr_reader :entry_spans, :resource_spans
32
+
33
+ def initialize(entry_spans:, resource_spans:)
34
+ @entry_spans = entry_spans.freeze
35
+ @resource_spans = resource_spans.freeze
36
+ @entries_by_key = @entry_spans.group_by(&:key)
37
+ end
38
+
39
+ def span_for(key)
40
+ @entries_by_key[key]&.first
41
+ end
42
+
43
+ def base_key_at(line_number)
44
+ @resource_spans.find { |span| span.cover?(line_number) }&.base_key
45
+ end
46
+
47
+ def member_at(line_number, parent:)
48
+ matches = @entry_spans.select do |span|
49
+ span.base_key == parent && span.resource_type != :string && span.cover?(line_number)
50
+ end
51
+ keys = matches.map(&:key).uniq
52
+ keys.one? ? keys.first : nil
53
+ end
54
+ end
55
+
56
+ module_function
57
+
58
+ def base_key(key)
59
+ key.to_s.sub(/:[a-z]+$/, '').sub(/\[\d+\]$/, '')
60
+ end
61
+
62
+ def type_for(key, explicit: nil)
63
+ return explicit.to_sym if explicit
64
+ return :plural if key.to_s.match?(/:[a-z]+$/)
65
+ return :array if key.to_s.match?(/\[\d+\]$/)
66
+
67
+ :string
68
+ end
69
+
70
+ def composite_key(name, type:, quantity: nil, index: nil)
71
+ case type.to_sym
72
+ when :plural then "#{name}:#{quantity}"
73
+ when :array then "#{name}[#{index}]"
74
+ else name
75
+ end
76
+ end
77
+
78
+ def type_for_tag(tag)
79
+ TAG_TYPES[tag]
80
+ end
81
+
82
+ def tag_for_type(type)
83
+ TYPE_TAGS[type.to_sym]
84
+ end
85
+
86
+ def index(content)
87
+ Scanner.new(content).index
88
+ end
89
+
90
+ # Internal scanner that runs after XML syntax validation. It preserves line
91
+ # positions while masking comments and records parent plus child spans.
92
+ class Scanner
93
+ def initialize(content)
94
+ @content = content
95
+ @visible_content = mask_comments(content)
96
+ @line_starts = [0]
97
+ content.to_enum(:scan, /\n/).each { @line_starts << Regexp.last_match.end(0) }
98
+ end
99
+
100
+ def index
101
+ entry_spans = []
102
+ resource_spans = []
103
+ offset = 0
104
+
105
+ while (match = RESOURCE_OPEN_PATTERN.match(@visible_content, offset))
106
+ resource_end, closing_start = element_end(match, match[:tag])
107
+ span = resource_span(match, resource_end)
108
+ if span
109
+ resource_spans << span
110
+ if span.resource_type == :string
111
+ entry_spans << span
112
+ else
113
+ entry_spans.concat(item_spans(match.end(0), closing_start, span))
114
+ end
115
+ end
116
+ offset = [resource_end, match.end(0)].max
117
+ end
118
+
119
+ Index.new(entry_spans: entry_spans, resource_spans: resource_spans)
120
+ end
121
+
122
+ private
123
+
124
+ def resource_span(match, resource_end)
125
+ name = attribute(match[:attributes], 'name')
126
+ type = AndroidResource.type_for_tag(match[:tag])
127
+ return unless name && type
128
+
129
+ start_line = line_for(match.begin(0))
130
+ end_line = line_for([resource_end - 1, match.begin(0)].max)
131
+ Span.new(
132
+ key: name,
133
+ base_key: name,
134
+ resource_type: type,
135
+ start_line: start_line,
136
+ end_line: end_line,
137
+ parent_start_line: start_line,
138
+ parent_end_line: end_line
139
+ )
140
+ end
141
+
142
+ def item_spans(content_start, content_end, parent_span)
143
+ spans = []
144
+ offset = content_start
145
+ array_index = 0
146
+
147
+ while offset < content_end && (match = ITEM_OPEN_PATTERN.match(@visible_content, offset))
148
+ break if match.begin(0) >= content_end
149
+
150
+ item_end, = element_end(match, 'item', limit: content_end)
151
+ key = item_key(match, parent_span, array_index)
152
+ spans << child_span(key, match.begin(0), item_end, parent_span) if key
153
+ array_index += 1 if parent_span.resource_type == :array
154
+ offset = [item_end, match.end(0)].max
155
+ end
156
+ spans
157
+ end
158
+
159
+ def item_key(match, parent_span, array_index)
160
+ quantity = attribute(match[:attributes], 'quantity') if parent_span.resource_type == :plural
161
+ return if parent_span.resource_type == :plural && quantity.nil?
162
+
163
+ AndroidResource.composite_key(
164
+ parent_span.base_key,
165
+ type: parent_span.resource_type,
166
+ quantity: quantity,
167
+ index: array_index
168
+ )
169
+ end
170
+
171
+ def child_span(key, start_offset, end_offset, parent_span)
172
+ Span.new(
173
+ key: key,
174
+ base_key: parent_span.base_key,
175
+ resource_type: parent_span.resource_type,
176
+ start_line: line_for(start_offset),
177
+ end_line: line_for([end_offset - 1, start_offset].max),
178
+ parent_start_line: parent_span.start_line,
179
+ parent_end_line: parent_span.end_line
180
+ )
181
+ end
182
+
183
+ def element_end(opening_match, tag, limit: @visible_content.length)
184
+ return [opening_match.end(0), opening_match.end(0)] if opening_match[0].rstrip.end_with?('/>')
185
+
186
+ closing = %r{</#{Regexp.escape(tag)}\s*>}.match(@visible_content, opening_match.end(0))
187
+ return [opening_match.end(0), opening_match.end(0)] unless closing && closing.begin(0) < limit
188
+
189
+ [closing.end(0), closing.begin(0)]
190
+ end
191
+
192
+ def attribute(attributes, name)
193
+ attributes[/\b#{Regexp.escape(name)}\s*=\s*(["'])(.*?)\1/m, 2]
194
+ end
195
+
196
+ def line_for(offset)
197
+ next_line = @line_starts.bsearch_index { |line_start| line_start > offset }
198
+ next_line || @line_starts.length
199
+ end
200
+
201
+ def mask_comments(content)
202
+ content
203
+ .gsub(/<!--.*?-->/m) { |comment| comment.gsub(/[^\n]/, ' ') }
204
+ .gsub(/<!\[CDATA\[.*?\]\]>/m) { |cdata| cdata.gsub(/[^\n]/, ' ') }
205
+ end
206
+ end
207
+ private_constant :Scanner
208
+ end
209
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module I18nContextGenerator
4
+ # Shared parsing and decoding for Apple/Swift quoted string literals.
5
+ module AppleStringLiteral
6
+ BODY_PATTERN = '(?:\\\\.|[^"\\\\])*'
7
+ ASSIGNMENT_PATTERN = /\A\s*"(?<key>#{BODY_PATTERN})"\s*=/
8
+
9
+ module_function
10
+
11
+ def assignment_key(content)
12
+ match = ASSIGNMENT_PATTERN.match(content)
13
+ decode(match[:key]) if match
14
+ end
15
+
16
+ def decode(content)
17
+ return if content.nil?
18
+
19
+ content
20
+ .gsub('\\"', '"')
21
+ .gsub("\\'", "'")
22
+ .gsub('\\\\', '\\')
23
+ .gsub('\\n', "\n")
24
+ .gsub('\\r', "\r")
25
+ .gsub('\\t', "\t")
26
+ end
27
+ end
28
+ end
@@ -1,15 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'digest'
4
+ require 'fileutils'
5
+ require 'oj'
6
+ require 'tempfile'
7
+
3
8
  module I18nContextGenerator
4
9
  # File-based cache for LLM results, keyed by translation key and source context.
5
10
  class Cache
6
- CACHE_DIR = '.i18n-context-generator-cache'
11
+ DEFAULT_DIR = '.i18n-context-generator-cache'
12
+ CACHE_DIR = DEFAULT_DIR # Backward-compatible constant name.
7
13
  # Bump this when prompt format, search heuristics, or output schema change
8
- CACHE_VERSION = 'v2'
14
+ CACHE_VERSION = 'v5'
15
+
16
+ attr_reader :directory
9
17
 
10
- def initialize(enabled: true)
18
+ def initialize(enabled: true, directory: DEFAULT_DIR)
11
19
  @enabled = enabled
12
- FileUtils.mkdir_p(CACHE_DIR) if @enabled && !File.directory?(CACHE_DIR)
20
+ @directory = directory
21
+ FileUtils.mkdir_p(@directory) if @enabled && !File.directory?(@directory)
13
22
  end
14
23
 
15
24
  def get(key, text, context: nil)
@@ -26,15 +35,21 @@ module I18nContextGenerator
26
35
 
27
36
  def set(key, text, result, context: nil)
28
37
  return unless @enabled
38
+ return if result[:error] || result['error']
29
39
 
30
40
  path = cache_path(key, text, context)
31
- File.write(path, Oj.dump(result, indent: 2, mode: :compat))
41
+ write_atomically(path, Oj.dump(result, indent: 2, mode: :compat))
32
42
  rescue StandardError => e
33
43
  warn "Cache write error for #{key}: #{e.message}"
34
44
  end
35
45
 
36
46
  def clear
37
- FileUtils.rm_rf(CACHE_DIR) if File.directory?(CACHE_DIR)
47
+ return unless File.directory?(@directory)
48
+
49
+ Dir.children(@directory).grep(/\A(?:[a-f0-9]{32}(?:[a-f0-9]{32})?\.json|context-cache-.*\.tmp)\z/).each do |filename|
50
+ FileUtils.rm_f(File.join(@directory, filename))
51
+ end
52
+ Dir.rmdir(@directory) if safe_to_remove_empty_directory? && Dir.empty?(@directory)
38
53
  end
39
54
 
40
55
  private
@@ -42,8 +57,25 @@ module I18nContextGenerator
42
57
  def cache_path(key, text, context)
43
58
  # Include version, key, text, and context (match locations/code) in hash
44
59
  # so cache invalidates when source code usage changes
45
- hash = Digest::MD5.hexdigest("#{CACHE_VERSION}:#{key}:#{text}:#{context}")
46
- File.join(CACHE_DIR, "#{hash}.json")
60
+ hash = Digest::SHA256.hexdigest("#{CACHE_VERSION}:#{key}:#{text}:#{context}")
61
+ File.join(@directory, "#{hash}.json")
62
+ end
63
+
64
+ def write_atomically(path, contents)
65
+ tempfile = Tempfile.new(['context-cache-', '.tmp'], @directory)
66
+ tempfile.chmod(0o600)
67
+ tempfile.write(contents)
68
+ tempfile.flush
69
+ tempfile.fsync
70
+ tempfile.close
71
+ File.rename(tempfile.path, path)
72
+ ensure
73
+ tempfile&.close!
74
+ end
75
+
76
+ def safe_to_remove_empty_directory?
77
+ expanded = File.expand_path(@directory)
78
+ ![File.expand_path(File::SEPARATOR), Dir.home, Dir.pwd].include?(expanded)
47
79
  end
48
80
  end
49
81
  end