i18n-context-generator 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +163 -26
- data/lib/i18n_context_generator/android_resource.rb +209 -0
- data/lib/i18n_context_generator/apple_string_literal.rb +28 -0
- data/lib/i18n_context_generator/cache.rb +40 -8
- data/lib/i18n_context_generator/changed_location.rb +55 -0
- data/lib/i18n_context_generator/cli.rb +163 -76
- data/lib/i18n_context_generator/config/cli_values.rb +33 -0
- data/lib/i18n_context_generator/config/defaults.rb +35 -0
- data/lib/i18n_context_generator/config/schema.rb +225 -0
- data/lib/i18n_context_generator/config/serialization.rb +64 -0
- data/lib/i18n_context_generator/config/validation.rb +249 -0
- data/lib/i18n_context_generator/config.rb +293 -102
- data/lib/i18n_context_generator/context_extractor/cache_identity.rb +65 -0
- data/lib/i18n_context_generator/context_extractor/extraction_result.rb +46 -0
- data/lib/i18n_context_generator/context_extractor/run_logging.rb +45 -6
- data/lib/i18n_context_generator/context_extractor/source_entries.rb +103 -0
- data/lib/i18n_context_generator/context_extractor/source_filters.rb +51 -5
- data/lib/i18n_context_generator/context_extractor/translation_filters.rb +91 -0
- data/lib/i18n_context_generator/context_extractor/workflow.rb +118 -0
- data/lib/i18n_context_generator/context_extractor.rb +207 -216
- data/lib/i18n_context_generator/file_classifier.rb +72 -0
- data/lib/i18n_context_generator/generated_comment.rb +32 -0
- data/lib/i18n_context_generator/git_diff/xml_changes.rb +235 -0
- data/lib/i18n_context_generator/git_diff.rb +224 -95
- data/lib/i18n_context_generator/llm/anthropic.rb +65 -38
- data/lib/i18n_context_generator/llm/client.rb +294 -92
- data/lib/i18n_context_generator/llm/openai.rb +92 -51
- data/lib/i18n_context_generator/llm/openai_compatible.rb +20 -0
- data/lib/i18n_context_generator/llm/prompt_evidence.rb +47 -0
- data/lib/i18n_context_generator/llm/request_policy.rb +147 -0
- data/lib/i18n_context_generator/localization_syntax.rb +246 -0
- data/lib/i18n_context_generator/parsers/android_xml_parser.rb +47 -8
- data/lib/i18n_context_generator/parsers/base.rb +7 -4
- data/lib/i18n_context_generator/parsers/json_parser.rb +4 -0
- data/lib/i18n_context_generator/parsers/xcstrings_parser.rb +79 -0
- data/lib/i18n_context_generator/parsers/yaml_parser.rb +20 -7
- data/lib/i18n_context_generator/path_policy.rb +107 -0
- data/lib/i18n_context_generator/platform_validator.rb +24 -50
- data/lib/i18n_context_generator/run_metrics.rb +47 -0
- data/lib/i18n_context_generator/searcher/comment_masking.rb +115 -0
- data/lib/i18n_context_generator/searcher/match_filtering.rb +52 -0
- data/lib/i18n_context_generator/searcher/source_discovery.rb +109 -64
- data/lib/i18n_context_generator/searcher.rb +61 -205
- data/lib/i18n_context_generator/supplemental_context.rb +77 -0
- data/lib/i18n_context_generator/translation_comment_index.rb +121 -0
- data/lib/i18n_context_generator/version.rb +1 -1
- data/lib/i18n_context_generator/writers/android_xml_writer.rb +48 -18
- data/lib/i18n_context_generator/writers/atomic_file.rb +37 -0
- data/lib/i18n_context_generator/writers/csv_writer.rb +43 -18
- data/lib/i18n_context_generator/writers/helpers.rb +7 -29
- data/lib/i18n_context_generator/writers/json_writer.rb +31 -6
- data/lib/i18n_context_generator/writers/preview.rb +80 -0
- data/lib/i18n_context_generator/writers/result_serialization.rb +30 -0
- data/lib/i18n_context_generator/writers/strings_writer.rb +23 -12
- data/lib/i18n_context_generator/writers/swift_writer.rb +16 -54
- data/lib/i18n_context_generator/writers/xcstrings_writer.rb +57 -0
- data/lib/i18n_context_generator/xcstrings_document.rb +218 -0
- data/lib/i18n_context_generator/xml_scanner.rb +38 -0
- data/lib/i18n_context_generator.rb +20 -4
- metadata +59 -12
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6588c09e273cb74c40dba8cc26798e46b82b5d47e5c840f6e4609843bb1f5ed8
|
|
4
|
+
data.tar.gz: 9b3e309fd4030a99e065a2c7693240f09d205f9d95323a9744ecb3eaa1b239b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2a7b9df974f135375341c7c1a5f73e40575f10c92d11a72ecc976ecef9ae6b25aac32351f1468c79671c870d87c2df576664a96b32ef00128815fc4ff016de1c
|
|
7
|
+
data.tar.gz: 0cc9db25973ffe209dea8cb4197df899310320361f0a8da2e45fa597fc592867bea22cdd5ad5337907c18cbdf824e269955fe79550baedd23f0d919fdef73c00
|
data/README.md
CHANGED
|
@@ -2,17 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
`i18n-context-generator` generates translator-facing context for your existing localization keys. It reads translation files, finds where each key is used in app code, and asks an LLM to explain the string's UI role in plain language.
|
|
4
4
|
|
|
5
|
-
You can export the results as CSV or JSON, write them back into `.strings
|
|
5
|
+
You can export the results as CSV or JSON, write them back into `.strings`, `.xcstrings`, or `strings.xml`, or update Swift `comment:` arguments directly.
|
|
6
6
|
|
|
7
7
|
It is designed for mobile codebases. Each run must target either iOS or Android, not both.
|
|
8
8
|
|
|
9
9
|
## What It Does
|
|
10
10
|
|
|
11
|
-
- Parses `.strings`, `strings.xml`, `.json`, and `.yml` or `.yaml` translation files
|
|
11
|
+
- Parses `.strings`, `.xcstrings`, `strings.xml`, `.json`, and `.yml` or `.yaml` translation files
|
|
12
12
|
- Searches Swift, Objective-C, Kotlin, Java, and Android XML for matching usages
|
|
13
|
-
- Uses Anthropic or OpenAI
|
|
13
|
+
- Uses Anthropic, OpenAI, or an explicitly configured OpenAI-compatible endpoint to infer UI context
|
|
14
14
|
- Supports diff-based runs, key filters, and key ranges for incremental work
|
|
15
|
-
-
|
|
15
|
+
- Separates validation, planning, patch preview, and mutation into explicit workflow stages
|
|
16
|
+
- Applies best-effort redaction of likely secrets, URLs, and emails by default
|
|
16
17
|
- Optionally caches results to avoid repeating identical LLM work
|
|
17
18
|
|
|
18
19
|
## Installation
|
|
@@ -24,13 +25,12 @@ chmod +x exe/i18n-context-generator
|
|
|
24
25
|
|
|
25
26
|
## Quick Start
|
|
26
27
|
|
|
27
|
-
|
|
28
|
+
Plan what would be processed without calling an LLM:
|
|
28
29
|
|
|
29
30
|
```bash
|
|
30
|
-
bundle exec exe/i18n-context-generator
|
|
31
|
+
bundle exec exe/i18n-context-generator plan \
|
|
31
32
|
-t ios/MyApp/Resources/Localizable.strings \
|
|
32
|
-
-s ios/MyApp
|
|
33
|
-
--dry-run
|
|
33
|
+
-s ios/MyApp
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
For a real run, set a provider API key:
|
|
@@ -77,7 +77,7 @@ bundle exec exe/i18n-context-generator extract \
|
|
|
77
77
|
--write-back-to-code
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
-
Use `--output`, `--write-back`, or `--write-back-to-code` depending on where you want results to go. If you have both iOS and Android code in the same repository, run the tool separately for each platform.
|
|
80
|
+
Use `--output`, `--stdout`, `--write-back`, or `--write-back-to-code` depending on where you want results to go. `extract` uses the configured `workflow.stage`, which defaults to `apply`; the named workflow commands always select their corresponding stage explicitly. If you have both iOS and Android code in the same repository, run the tool separately for each platform.
|
|
81
81
|
|
|
82
82
|
## Configuration
|
|
83
83
|
|
|
@@ -91,35 +91,56 @@ bundle exec exe/i18n-context-generator extract --config .i18n-context-generator.
|
|
|
91
91
|
Example `.i18n-context-generator.yml`:
|
|
92
92
|
|
|
93
93
|
```yaml
|
|
94
|
+
schema_version: 1
|
|
95
|
+
|
|
94
96
|
translations:
|
|
95
97
|
- path: ios/MyApp/Resources/Localizable.strings
|
|
96
98
|
|
|
97
99
|
source:
|
|
98
100
|
paths:
|
|
99
101
|
- ios/MyApp
|
|
102
|
+
# Optional additions to the built-in dependency, build, and test ignores
|
|
100
103
|
ignore:
|
|
101
|
-
- "
|
|
102
|
-
|
|
103
|
-
|
|
104
|
+
- "**/*.generated.*"
|
|
105
|
+
|
|
106
|
+
context:
|
|
107
|
+
# Free-form text or Markdown included in full for every selected key.
|
|
108
|
+
files:
|
|
109
|
+
- GLOSSARY.md
|
|
110
|
+
- docs/localization-style.md
|
|
104
111
|
|
|
105
112
|
llm:
|
|
106
113
|
provider: anthropic
|
|
107
|
-
model
|
|
114
|
+
# Optional; each provider has its own default model.
|
|
115
|
+
# model: claude-sonnet-4-6
|
|
116
|
+
# openai_compatible requires an explicit model and endpoint.
|
|
117
|
+
# endpoint: http://127.0.0.1:11434/v1/responses
|
|
108
118
|
|
|
109
119
|
processing:
|
|
110
120
|
concurrency: 5
|
|
111
121
|
context_lines: 15
|
|
112
122
|
max_matches_per_key: 3
|
|
123
|
+
max_prompt_chars: 50000
|
|
124
|
+
|
|
125
|
+
cache:
|
|
126
|
+
enabled: false
|
|
127
|
+
directory: .i18n-context-generator-cache
|
|
128
|
+
|
|
129
|
+
workflow:
|
|
130
|
+
stage: apply
|
|
113
131
|
|
|
114
132
|
output:
|
|
115
133
|
format: csv
|
|
116
134
|
path: translation-context.csv
|
|
135
|
+
stdout: false
|
|
117
136
|
write_back: false
|
|
118
137
|
write_back_to_code: false
|
|
119
138
|
context_prefix: "Context: "
|
|
120
139
|
context_mode: replace
|
|
121
140
|
|
|
122
141
|
swift:
|
|
142
|
+
# These syntaxes are used by extraction and Swift comment write-back.
|
|
143
|
+
# Custom entries extend the built-in defaults shown below.
|
|
123
144
|
functions:
|
|
124
145
|
- NSLocalizedString
|
|
125
146
|
- "String(localized:"
|
|
@@ -128,48 +149,96 @@ swift:
|
|
|
128
149
|
privacy:
|
|
129
150
|
include_file_paths: false
|
|
130
151
|
include_translation_comments: true
|
|
152
|
+
# Best-effort only; source snippets are still sent to the remote provider.
|
|
131
153
|
redact_prompts: true
|
|
132
154
|
```
|
|
133
155
|
|
|
134
156
|
Use a separate config for Android instead of mixing iOS and Android paths in the same run.
|
|
157
|
+
Client `source.ignore` entries extend the built-in ignore list; they do not replace it.
|
|
158
|
+
Configured `swift.functions` extend the built-in defaults and are used consistently for usage search, source-first discovery, and `--write-back-to-code`. A custom function may be written as either `MyLocalizedString` or `MyLocalizedString(`; its first string argument, with or without a label such as `key:`, is treated as the translation key.
|
|
159
|
+
Unversioned configuration files use schema version 1 compatibility mode: unknown options produce warnings and remain ignored as they were before versioning. New files should declare `schema_version: 1`; explicit versioning enables strict unknown-option validation, and future versions fail clearly. Run `i18n-context-generator config validate [PATH]` to validate a file or add `--print-config` to an extraction command to inspect the fully resolved, secret-free configuration.
|
|
135
160
|
|
|
136
161
|
## CLI Reference
|
|
137
162
|
|
|
138
163
|
### Inputs and Output
|
|
139
164
|
|
|
140
|
-
- `-t`, `--
|
|
141
|
-
- `-s`, `--source
|
|
165
|
+
- `-t`, `--translation FILE`: translation file to process; repeat for multiple files
|
|
166
|
+
- `-s`, `--source PATH`: source file or directory to search; repeat for multiple paths
|
|
167
|
+
- `--context-file PATH`: free-form supplemental context file; repeat for multiple files
|
|
142
168
|
- `-c`, `--config PATH`: load options from `.i18n-context-generator.yml`
|
|
143
169
|
- `-o`, `--output PATH`: write results to a file
|
|
170
|
+
- `--stdout`: write structured CSV or JSON to standard output
|
|
144
171
|
- `-f`, `--format csv|json`: output format, default `csv`
|
|
172
|
+
- `--print-config`: print the resolved configuration and exit
|
|
145
173
|
|
|
146
174
|
### LLM Settings
|
|
147
175
|
|
|
148
|
-
- `-p`, `--provider anthropic|openai`: LLM provider, default `anthropic`
|
|
176
|
+
- `-p`, `--provider anthropic|openai|openai_compatible`: LLM provider, default `anthropic`
|
|
149
177
|
- `-m`, `--model MODEL`: explicit model override
|
|
178
|
+
- `--endpoint URL`: explicit Responses API endpoint for `openai_compatible`
|
|
150
179
|
- `--concurrency N`: parallel request count, default `5`
|
|
180
|
+
- `--max-prompt-chars N`: hard character limit per LLM prompt, default `50000`
|
|
151
181
|
|
|
152
182
|
### Filtering and Incremental Runs
|
|
153
183
|
|
|
154
|
-
- `-k`, `--
|
|
184
|
+
- `-k`, `--key PATTERN`: wildcard key filter such as `settings.*`; repeat for multiple patterns
|
|
155
185
|
- `--diff-base REF`: process only keys changed since a Git ref
|
|
186
|
+
- `--diff-head REF`: compare `--diff-base` to this ref, default `HEAD`
|
|
156
187
|
- `--start-key KEY`: start at a specific key, inclusive
|
|
157
188
|
- `--end-key KEY`: stop at a specific key, inclusive
|
|
158
189
|
- `--dry-run`: preview matching keys without calling the LLM
|
|
159
190
|
|
|
160
191
|
### Write-Back and Prompt Controls
|
|
161
192
|
|
|
162
|
-
- `--write-back`: update `.strings
|
|
193
|
+
- `--write-back`: update `.strings`, `.xcstrings`, or `strings.xml`
|
|
163
194
|
- `--write-back-to-code`: update Swift `comment:` arguments
|
|
164
195
|
- `--context-prefix TEXT`: prefix generated comments, default `Context: `
|
|
165
196
|
- `--context-mode replace|append`: replace existing comments or append to them
|
|
166
197
|
- `--cache`: enable on-disk caching
|
|
198
|
+
- `--cache-dir PATH`: cache directory, default `.i18n-context-generator-cache`
|
|
167
199
|
- `--include-file-paths`: include full source paths in prompts
|
|
168
200
|
- `--include-translation-comments`: include existing translation comments in prompts, default `true`
|
|
169
|
-
- `--redact-prompts`: redact likely secrets and PII before prompts are sent, default `true`
|
|
201
|
+
- `--redact-prompts`: best-effort redact likely secrets and PII before prompts are sent, default `true`
|
|
170
202
|
|
|
171
203
|
Run `bundle exec exe/i18n-context-generator help extract` for the full command reference.
|
|
172
204
|
|
|
205
|
+
The legacy `--translations` and `--keys` flags still accept comma-separated lists with a deprecation warning. Repeatable singular flags always preserve literal commas in paths and key patterns.
|
|
206
|
+
|
|
207
|
+
### Workflow stages
|
|
208
|
+
|
|
209
|
+
- `check`: validate configuration and confirm source usage without calling the LLM
|
|
210
|
+
- `plan`: list selected keys and destinations without calling the LLM
|
|
211
|
+
- `preview-diff`: generate context and print the exact write-back patch to stdout
|
|
212
|
+
without changing translation, source, or structured-output files; diagnostics
|
|
213
|
+
go to stderr
|
|
214
|
+
- `apply`: generate context and write to the configured destinations
|
|
215
|
+
- `extract`: compatibility entry point that uses `workflow.stage` from configuration, defaulting to `apply`
|
|
216
|
+
|
|
217
|
+
`--dry-run` remains available as a compatibility shortcut for non-mutating key selection. Use `preview-diff` when you need to inspect generated write-back content rather than only the selected keys.
|
|
218
|
+
|
|
219
|
+
### Prompt privacy and size
|
|
220
|
+
|
|
221
|
+
Remote-provider runs send translation text, selected source snippets, configured context files, and programmatic supplemental context off the machine. By default, full paths are reduced to basenames and the tool applies pattern-based redaction to keys, text, comments, paths, scopes, matched lines, surrounding context, and supplemental context. This is a best-effort safeguard, not a guarantee that all private or identifying data will be detected. Review every input you configure before using a remote provider.
|
|
222
|
+
|
|
223
|
+
All context files are treated as opaque, untrusted evidence: they are neither parsed as a glossary nor filtered for relevance, and their complete contents are included in every selected key's request. Named runtime context follows the same contract. Evidence cannot override the system prompt or instruct the model, but prompt-injection defenses are not a substitute for reviewing sensitive inputs.
|
|
224
|
+
|
|
225
|
+
`processing.max_prompt_chars` bounds each user prompt. When evidence exceeds the limit, surrounding code and optional metadata are truncated deterministically, with the translation key, text, and at least one usage receiving priority. Supplemental context is never partially included: if all of it cannot fit alongside the minimum application evidence, extraction fails with guidance to reduce the context or raise the limit. Provider output is validated against the application contract before it can reach an output writer.
|
|
226
|
+
|
|
227
|
+
For a local or third-party Responses API, select `openai_compatible` and configure both `llm.model` and `llm.endpoint`. Plain HTTP is accepted only for loopback endpoints; remote endpoints must use HTTPS. Authentication is isolated to `OPENAI_COMPATIBLE_API_KEY`, so this provider never implicitly forwards `OPENAI_API_KEY` to a custom host.
|
|
228
|
+
|
|
229
|
+
```yaml
|
|
230
|
+
llm:
|
|
231
|
+
provider: openai_compatible
|
|
232
|
+
model: local-model-name
|
|
233
|
+
endpoint: http://127.0.0.1:11434/v1/responses
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### Cache behavior
|
|
237
|
+
|
|
238
|
+
Caching is opt-in and stores successful results only; provider, resolved model, custom endpoint, prompt controls, source context, source-discovery locations, and supplemental-context content digests are part of each cache identity. Raw context-file and runtime-context contents are not stored in the cache identity. Writes use private temporary files and atomic replacement so concurrent workers cannot leave partial JSON behind.
|
|
239
|
+
|
|
240
|
+
The default directory is `.i18n-context-generator-cache`. Add that directory—or your custom `cache.directory`—to the client repository's `.gitignore`; cached LLM output should not be committed. The generated starter configuration keeps caching disabled by default.
|
|
241
|
+
|
|
173
242
|
## Supported Inputs
|
|
174
243
|
|
|
175
244
|
### Translation Files
|
|
@@ -177,15 +246,24 @@ Run `bundle exec exe/i18n-context-generator help extract` for the full command r
|
|
|
177
246
|
| Format | Notes |
|
|
178
247
|
|--------|-------|
|
|
179
248
|
| `.strings` | Apple strings files |
|
|
249
|
+
| `.xcstrings` | Apple string catalogs; reads the source language, comments, and plural variations |
|
|
180
250
|
| `strings.xml` | Android string resources, including plurals and arrays |
|
|
181
|
-
| `.json` |
|
|
182
|
-
| `.yml`, `.yaml` |
|
|
251
|
+
| `.json` | Read-only hydration source; nested keys are flattened |
|
|
252
|
+
| `.yml`, `.yaml` | Read-only hydration source; nested keys are flattened and locale roots must be configured explicitly |
|
|
253
|
+
|
|
254
|
+
For Rails-style YAML files with a locale root, name that root in the translation entry. Without `locale:`, the top-level key is retained as part of every translation key; locale-shaped application namespaces are never guessed or removed.
|
|
255
|
+
|
|
256
|
+
```yaml
|
|
257
|
+
translations:
|
|
258
|
+
- path: config/locales/en.yml
|
|
259
|
+
locale: en
|
|
260
|
+
```
|
|
183
261
|
|
|
184
262
|
### Source Search
|
|
185
263
|
|
|
186
264
|
| Platform | Files searched | Typical patterns |
|
|
187
265
|
|----------|----------------|------------------|
|
|
188
|
-
| iOS | `.swift`, `.m`, `.mm`, `.h` | `NSLocalizedString`, `String(localized:)`, `LocalizedStringKey`, `Text`, `.localized` |
|
|
266
|
+
| iOS | `.swift`, `.m`, `.mm`, `.h` | `NSLocalizedString`, `String(localized:)`, `LocalizedStringResource`, `LocalizedStringKey`, `Text`, `.localized` |
|
|
189
267
|
| Android | `.kt`, `.java`, `.xml` | `R.string.*`, `getString(...)`, `stringResource(...)`, `@string/...`, plurals, arrays |
|
|
190
268
|
|
|
191
269
|
## Output
|
|
@@ -193,12 +271,19 @@ Run `bundle exec exe/i18n-context-generator help extract` for the full command r
|
|
|
193
271
|
CSV example:
|
|
194
272
|
|
|
195
273
|
```csv
|
|
196
|
-
key,text,description,ui_element,tone,max_length,locations,error
|
|
197
|
-
settings.title,Settings,Navigation bar title for the main settings screen,navigation,neutral,15,ios/SettingsViewController.swift:17,
|
|
198
|
-
common.save,Save,Primary action button in forms and edit screens,button,neutral,10,ios/ProfileViewController.swift:31,
|
|
199
|
-
error.network,Unable to connect,Error message shown when network requests fail,alert,apologetic,,ios/ProfileViewController.swift:94,
|
|
274
|
+
key,text,description,ui_element,tone,max_length,confidence,ambiguity_reason,locations,status,cache_hit,request_count,input_tokens,output_tokens,retries,error
|
|
275
|
+
settings.title,Settings,Navigation bar title for the main settings screen,navigation,neutral,15,high,,ios/SettingsViewController.swift:17,success,false,1,834,61,0,
|
|
200
276
|
```
|
|
201
277
|
|
|
278
|
+
JSON and CSV output schema version 1 includes source-file and translation-key
|
|
279
|
+
identity, complete and changed-only evidence locations, side-aware translation
|
|
280
|
+
diff locations, confidence, ambiguity, token counts, retries, and request count.
|
|
281
|
+
This keeps duplicate keys from different translation files distinguishable. The
|
|
282
|
+
run summary reports aggregate requests, cache hits, tokens, retries, and an
|
|
283
|
+
estimated list-price cost when the selected model has known pricing. Cost is
|
|
284
|
+
informational and does not account for provider-specific discounts or billing
|
|
285
|
+
adjustments.
|
|
286
|
+
|
|
202
287
|
With `--write-back`, generated context is written back into translation files:
|
|
203
288
|
|
|
204
289
|
**Before**
|
|
@@ -251,6 +336,58 @@ bundle exec exe/i18n-context-generator extract \
|
|
|
251
336
|
--context-prefix ""
|
|
252
337
|
```
|
|
253
338
|
|
|
339
|
+
`--diff-base` compares against `HEAD` by default. Pass `--diff-head` when a CI system prepares explicit refs:
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
bundle exec exe/i18n-context-generator extract \
|
|
343
|
+
-t ios/Resources/Localizable.strings \
|
|
344
|
+
-s ios \
|
|
345
|
+
--diff-base danger_base \
|
|
346
|
+
--diff-head danger_head \
|
|
347
|
+
--output translation-context.json
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
Programmatic callers can use the same range and may pass enum-style options as strings or symbols:
|
|
351
|
+
|
|
352
|
+
```ruby
|
|
353
|
+
config = I18nContextGenerator::Config.new(
|
|
354
|
+
translations: ['ios/Resources/Localizable.strings'],
|
|
355
|
+
source_paths: ['ios'],
|
|
356
|
+
context_files: ['GLOSSARY.md', 'docs/localization-style.md'],
|
|
357
|
+
supplemental_context: {
|
|
358
|
+
'Pull request title' => 'Clarify Reader renewal labels',
|
|
359
|
+
'Pull request description' => 'Updates subscription-expiry messaging.'
|
|
360
|
+
},
|
|
361
|
+
discovery_mode: :translations,
|
|
362
|
+
provider: :anthropic,
|
|
363
|
+
diff_base: 'danger_base',
|
|
364
|
+
diff_head: 'danger_head'
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
extractor = I18nContextGenerator::ContextExtractor.new(
|
|
368
|
+
config,
|
|
369
|
+
quiet: true,
|
|
370
|
+
progress: false
|
|
371
|
+
)
|
|
372
|
+
extractor.run
|
|
373
|
+
|
|
374
|
+
extractor.results.each do |result|
|
|
375
|
+
result.status # :success, :no_usage, or :error
|
|
376
|
+
result.actionable? # Safe to present to a user
|
|
377
|
+
result.locations # Every source location used as extraction evidence
|
|
378
|
+
result.changed_locations # Source evidence changed inside the configured range
|
|
379
|
+
result.changed_location_groups # Changed source lines grouped by localization occurrence
|
|
380
|
+
result.translation_key # Owning translation resource (including Android collections)
|
|
381
|
+
# ChangedLocation objects with file, line, side (:left/:right), and an
|
|
382
|
+
# optional right-side fallback_line for review suggestions.
|
|
383
|
+
result.changed_translation_locations
|
|
384
|
+
end
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
An invalid ref or failed `git diff` raises `I18nContextGenerator::Error`; it is never reported as an unchanged range.
|
|
388
|
+
Programmatic callers may also inject `log_output:`, `structured_output:`, and
|
|
389
|
+
`patch_output:` streams instead of using the process-wide standard streams.
|
|
390
|
+
|
|
254
391
|
Example GitHub Actions step:
|
|
255
392
|
|
|
256
393
|
```yaml
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module I18nContextGenerator
|
|
4
|
+
# Canonical Android string-resource identity and source-span indexing.
|
|
5
|
+
module AndroidResource
|
|
6
|
+
TAG_TYPES = {
|
|
7
|
+
'string' => :string,
|
|
8
|
+
'plurals' => :plural,
|
|
9
|
+
'string-array' => :array
|
|
10
|
+
}.freeze
|
|
11
|
+
TYPE_TAGS = TAG_TYPES.invert.freeze
|
|
12
|
+
RESOURCE_OPEN_PATTERN = /<(?<tag>string-array|plurals|string)\b(?<attributes>[^>]*?)>/m
|
|
13
|
+
ITEM_OPEN_PATTERN = /<item\b(?<attributes>[^>]*?)>/m
|
|
14
|
+
|
|
15
|
+
Span = Data.define(:key, :base_key, :resource_type, :start_line, :end_line, :parent_start_line, :parent_end_line) do
|
|
16
|
+
def line_span
|
|
17
|
+
start_line..end_line
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def parent_line_span
|
|
21
|
+
parent_start_line..parent_end_line
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def cover?(line_number)
|
|
25
|
+
line_span.cover?(line_number)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Query object shared by parsers and diff/location consumers.
|
|
30
|
+
class Index
|
|
31
|
+
attr_reader :entry_spans, :resource_spans
|
|
32
|
+
|
|
33
|
+
def initialize(entry_spans:, resource_spans:)
|
|
34
|
+
@entry_spans = entry_spans.freeze
|
|
35
|
+
@resource_spans = resource_spans.freeze
|
|
36
|
+
@entries_by_key = @entry_spans.group_by(&:key)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def span_for(key)
|
|
40
|
+
@entries_by_key[key]&.first
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def base_key_at(line_number)
|
|
44
|
+
@resource_spans.find { |span| span.cover?(line_number) }&.base_key
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def member_at(line_number, parent:)
|
|
48
|
+
matches = @entry_spans.select do |span|
|
|
49
|
+
span.base_key == parent && span.resource_type != :string && span.cover?(line_number)
|
|
50
|
+
end
|
|
51
|
+
keys = matches.map(&:key).uniq
|
|
52
|
+
keys.one? ? keys.first : nil
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
module_function
|
|
57
|
+
|
|
58
|
+
def base_key(key)
|
|
59
|
+
key.to_s.sub(/:[a-z]+$/, '').sub(/\[\d+\]$/, '')
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def type_for(key, explicit: nil)
|
|
63
|
+
return explicit.to_sym if explicit
|
|
64
|
+
return :plural if key.to_s.match?(/:[a-z]+$/)
|
|
65
|
+
return :array if key.to_s.match?(/\[\d+\]$/)
|
|
66
|
+
|
|
67
|
+
:string
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def composite_key(name, type:, quantity: nil, index: nil)
|
|
71
|
+
case type.to_sym
|
|
72
|
+
when :plural then "#{name}:#{quantity}"
|
|
73
|
+
when :array then "#{name}[#{index}]"
|
|
74
|
+
else name
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def type_for_tag(tag)
|
|
79
|
+
TAG_TYPES[tag]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def tag_for_type(type)
|
|
83
|
+
TYPE_TAGS[type.to_sym]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def index(content)
|
|
87
|
+
Scanner.new(content).index
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Internal scanner that runs after XML syntax validation. It preserves line
|
|
91
|
+
# positions while masking comments and records parent plus child spans.
|
|
92
|
+
class Scanner
|
|
93
|
+
def initialize(content)
|
|
94
|
+
@content = content
|
|
95
|
+
@visible_content = mask_comments(content)
|
|
96
|
+
@line_starts = [0]
|
|
97
|
+
content.to_enum(:scan, /\n/).each { @line_starts << Regexp.last_match.end(0) }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def index
|
|
101
|
+
entry_spans = []
|
|
102
|
+
resource_spans = []
|
|
103
|
+
offset = 0
|
|
104
|
+
|
|
105
|
+
while (match = RESOURCE_OPEN_PATTERN.match(@visible_content, offset))
|
|
106
|
+
resource_end, closing_start = element_end(match, match[:tag])
|
|
107
|
+
span = resource_span(match, resource_end)
|
|
108
|
+
if span
|
|
109
|
+
resource_spans << span
|
|
110
|
+
if span.resource_type == :string
|
|
111
|
+
entry_spans << span
|
|
112
|
+
else
|
|
113
|
+
entry_spans.concat(item_spans(match.end(0), closing_start, span))
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
offset = [resource_end, match.end(0)].max
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
Index.new(entry_spans: entry_spans, resource_spans: resource_spans)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
private
|
|
123
|
+
|
|
124
|
+
def resource_span(match, resource_end)
|
|
125
|
+
name = attribute(match[:attributes], 'name')
|
|
126
|
+
type = AndroidResource.type_for_tag(match[:tag])
|
|
127
|
+
return unless name && type
|
|
128
|
+
|
|
129
|
+
start_line = line_for(match.begin(0))
|
|
130
|
+
end_line = line_for([resource_end - 1, match.begin(0)].max)
|
|
131
|
+
Span.new(
|
|
132
|
+
key: name,
|
|
133
|
+
base_key: name,
|
|
134
|
+
resource_type: type,
|
|
135
|
+
start_line: start_line,
|
|
136
|
+
end_line: end_line,
|
|
137
|
+
parent_start_line: start_line,
|
|
138
|
+
parent_end_line: end_line
|
|
139
|
+
)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def item_spans(content_start, content_end, parent_span)
|
|
143
|
+
spans = []
|
|
144
|
+
offset = content_start
|
|
145
|
+
array_index = 0
|
|
146
|
+
|
|
147
|
+
while offset < content_end && (match = ITEM_OPEN_PATTERN.match(@visible_content, offset))
|
|
148
|
+
break if match.begin(0) >= content_end
|
|
149
|
+
|
|
150
|
+
item_end, = element_end(match, 'item', limit: content_end)
|
|
151
|
+
key = item_key(match, parent_span, array_index)
|
|
152
|
+
spans << child_span(key, match.begin(0), item_end, parent_span) if key
|
|
153
|
+
array_index += 1 if parent_span.resource_type == :array
|
|
154
|
+
offset = [item_end, match.end(0)].max
|
|
155
|
+
end
|
|
156
|
+
spans
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def item_key(match, parent_span, array_index)
|
|
160
|
+
quantity = attribute(match[:attributes], 'quantity') if parent_span.resource_type == :plural
|
|
161
|
+
return if parent_span.resource_type == :plural && quantity.nil?
|
|
162
|
+
|
|
163
|
+
AndroidResource.composite_key(
|
|
164
|
+
parent_span.base_key,
|
|
165
|
+
type: parent_span.resource_type,
|
|
166
|
+
quantity: quantity,
|
|
167
|
+
index: array_index
|
|
168
|
+
)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def child_span(key, start_offset, end_offset, parent_span)
|
|
172
|
+
Span.new(
|
|
173
|
+
key: key,
|
|
174
|
+
base_key: parent_span.base_key,
|
|
175
|
+
resource_type: parent_span.resource_type,
|
|
176
|
+
start_line: line_for(start_offset),
|
|
177
|
+
end_line: line_for([end_offset - 1, start_offset].max),
|
|
178
|
+
parent_start_line: parent_span.start_line,
|
|
179
|
+
parent_end_line: parent_span.end_line
|
|
180
|
+
)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def element_end(opening_match, tag, limit: @visible_content.length)
|
|
184
|
+
return [opening_match.end(0), opening_match.end(0)] if opening_match[0].rstrip.end_with?('/>')
|
|
185
|
+
|
|
186
|
+
closing = %r{</#{Regexp.escape(tag)}\s*>}.match(@visible_content, opening_match.end(0))
|
|
187
|
+
return [opening_match.end(0), opening_match.end(0)] unless closing && closing.begin(0) < limit
|
|
188
|
+
|
|
189
|
+
[closing.end(0), closing.begin(0)]
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def attribute(attributes, name)
|
|
193
|
+
attributes[/\b#{Regexp.escape(name)}\s*=\s*(["'])(.*?)\1/m, 2]
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def line_for(offset)
|
|
197
|
+
next_line = @line_starts.bsearch_index { |line_start| line_start > offset }
|
|
198
|
+
next_line || @line_starts.length
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def mask_comments(content)
|
|
202
|
+
content
|
|
203
|
+
.gsub(/<!--.*?-->/m) { |comment| comment.gsub(/[^\n]/, ' ') }
|
|
204
|
+
.gsub(/<!\[CDATA\[.*?\]\]>/m) { |cdata| cdata.gsub(/[^\n]/, ' ') }
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
private_constant :Scanner
|
|
208
|
+
end
|
|
209
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module I18nContextGenerator
|
|
4
|
+
# Shared parsing and decoding for Apple/Swift quoted string literals.
|
|
5
|
+
module AppleStringLiteral
|
|
6
|
+
BODY_PATTERN = '(?:\\\\.|[^"\\\\])*'
|
|
7
|
+
ASSIGNMENT_PATTERN = /\A\s*"(?<key>#{BODY_PATTERN})"\s*=/
|
|
8
|
+
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def assignment_key(content)
|
|
12
|
+
match = ASSIGNMENT_PATTERN.match(content)
|
|
13
|
+
decode(match[:key]) if match
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def decode(content)
|
|
17
|
+
return if content.nil?
|
|
18
|
+
|
|
19
|
+
content
|
|
20
|
+
.gsub('\\"', '"')
|
|
21
|
+
.gsub("\\'", "'")
|
|
22
|
+
.gsub('\\\\', '\\')
|
|
23
|
+
.gsub('\\n', "\n")
|
|
24
|
+
.gsub('\\r', "\r")
|
|
25
|
+
.gsub('\\t', "\t")
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'oj'
|
|
6
|
+
require 'tempfile'
|
|
7
|
+
|
|
3
8
|
module I18nContextGenerator
|
|
4
9
|
# File-based cache for LLM results, keyed by translation key and source context.
|
|
5
10
|
class Cache
|
|
6
|
-
|
|
11
|
+
DEFAULT_DIR = '.i18n-context-generator-cache'
|
|
12
|
+
CACHE_DIR = DEFAULT_DIR # Backward-compatible constant name.
|
|
7
13
|
# Bump this when prompt format, search heuristics, or output schema change
|
|
8
|
-
CACHE_VERSION = '
|
|
14
|
+
CACHE_VERSION = 'v5'
|
|
15
|
+
|
|
16
|
+
attr_reader :directory
|
|
9
17
|
|
|
10
|
-
def initialize(enabled: true)
|
|
18
|
+
def initialize(enabled: true, directory: DEFAULT_DIR)
|
|
11
19
|
@enabled = enabled
|
|
12
|
-
|
|
20
|
+
@directory = directory
|
|
21
|
+
FileUtils.mkdir_p(@directory) if @enabled && !File.directory?(@directory)
|
|
13
22
|
end
|
|
14
23
|
|
|
15
24
|
def get(key, text, context: nil)
|
|
@@ -26,15 +35,21 @@ module I18nContextGenerator
|
|
|
26
35
|
|
|
27
36
|
def set(key, text, result, context: nil)
|
|
28
37
|
return unless @enabled
|
|
38
|
+
return if result[:error] || result['error']
|
|
29
39
|
|
|
30
40
|
path = cache_path(key, text, context)
|
|
31
|
-
|
|
41
|
+
write_atomically(path, Oj.dump(result, indent: 2, mode: :compat))
|
|
32
42
|
rescue StandardError => e
|
|
33
43
|
warn "Cache write error for #{key}: #{e.message}"
|
|
34
44
|
end
|
|
35
45
|
|
|
36
46
|
def clear
|
|
37
|
-
|
|
47
|
+
return unless File.directory?(@directory)
|
|
48
|
+
|
|
49
|
+
Dir.children(@directory).grep(/\A(?:[a-f0-9]{32}(?:[a-f0-9]{32})?\.json|context-cache-.*\.tmp)\z/).each do |filename|
|
|
50
|
+
FileUtils.rm_f(File.join(@directory, filename))
|
|
51
|
+
end
|
|
52
|
+
Dir.rmdir(@directory) if safe_to_remove_empty_directory? && Dir.empty?(@directory)
|
|
38
53
|
end
|
|
39
54
|
|
|
40
55
|
private
|
|
@@ -42,8 +57,25 @@ module I18nContextGenerator
|
|
|
42
57
|
def cache_path(key, text, context)
|
|
43
58
|
# Include version, key, text, and context (match locations/code) in hash
|
|
44
59
|
# so cache invalidates when source code usage changes
|
|
45
|
-
hash = Digest::
|
|
46
|
-
File.join(
|
|
60
|
+
hash = Digest::SHA256.hexdigest("#{CACHE_VERSION}:#{key}:#{text}:#{context}")
|
|
61
|
+
File.join(@directory, "#{hash}.json")
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def write_atomically(path, contents)
|
|
65
|
+
tempfile = Tempfile.new(['context-cache-', '.tmp'], @directory)
|
|
66
|
+
tempfile.chmod(0o600)
|
|
67
|
+
tempfile.write(contents)
|
|
68
|
+
tempfile.flush
|
|
69
|
+
tempfile.fsync
|
|
70
|
+
tempfile.close
|
|
71
|
+
File.rename(tempfile.path, path)
|
|
72
|
+
ensure
|
|
73
|
+
tempfile&.close!
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def safe_to_remove_empty_directory?
|
|
77
|
+
expanded = File.expand_path(@directory)
|
|
78
|
+
![File.expand_path(File::SEPARATOR), Dir.home, Dir.pwd].include?(expanded)
|
|
47
79
|
end
|
|
48
80
|
end
|
|
49
81
|
end
|