locallingo 0.4.3 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +74 -3
- data/README.md +94 -0
- data/config/locallingo.default.yml +11 -1
- data/lib/locallingo/active_record.rb +148 -0
- data/lib/locallingo/batch_translator.rb +151 -0
- data/lib/locallingo/cli.rb +44 -10
- data/lib/locallingo/configuration.rb +4 -1
- data/lib/locallingo/content_leaves.rb +40 -0
- data/lib/locallingo/drift.rb +93 -0
- data/lib/locallingo/json_extraction.rb +14 -2
- data/lib/locallingo/manager.rb +68 -121
- data/lib/locallingo/prompt.rb +73 -0
- data/lib/locallingo/providers/ruby_llm.rb +36 -9
- data/lib/locallingo/quality_checker.rb +31 -6
- data/lib/locallingo/record_translator.rb +91 -0
- data/lib/locallingo/translator.rb +70 -0
- data/lib/locallingo/validators/duplicate_values.rb +4 -4
- data/lib/locallingo/validators/manual_edits.rb +4 -8
- data/lib/locallingo/validators/outdated.rb +5 -5
- data/lib/locallingo/version.rb +1 -1
- data/lib/locallingo.rb +5 -0
- metadata +10 -3
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Locallingo
|
|
4
|
+
# Walks a nested Hash/Array value to its leaves, keyed by a KeyFlattener-style
|
|
5
|
+
# path (`steps[1].text`, `table[0][1]`). Unlike KeyFlattener, strings inside
|
|
6
|
+
# nested arrays count, and each leaf carries the Hash keys and Array indices
|
|
7
|
+
# leading to it, so a value can be set back in place without parsing paths.
|
|
8
|
+
module ContentLeaves
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# `path => [parts, value]`. Two leaves with the same path (a literal "a.b"
|
|
12
|
+
# key next to a → b) raise: one would be silently dropped from translation.
|
|
13
|
+
def call(value, label: "content")
|
|
14
|
+
collect(value, [], {}, label)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def collect(value, parts, leaves, label)
|
|
18
|
+
case value
|
|
19
|
+
when Hash then value.each { |key, item| collect(item, parts + [key.to_s], leaves, label) }
|
|
20
|
+
when Array then value.each_with_index { |item, index| collect(item, parts + [index], leaves, label) }
|
|
21
|
+
else
|
|
22
|
+
path = path(parts)
|
|
23
|
+
raise Error, "Ambiguous translation path #{path.inspect} in #{label}" if leaves.key?(path)
|
|
24
|
+
|
|
25
|
+
leaves[path] = [parts, value]
|
|
26
|
+
end
|
|
27
|
+
leaves
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def path(parts)
|
|
31
|
+
parts.each_with_index.map do |part, index|
|
|
32
|
+
next "[#{part}]" if part.is_a?(Integer)
|
|
33
|
+
|
|
34
|
+
index.zero? ? part : ".#{part}"
|
|
35
|
+
end.join
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private_class_method :collect, :path
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "state_store"
|
|
5
|
+
|
|
6
|
+
module Locallingo
|
|
7
|
+
# Missing / outdated / hand-edited decisions over one value and its state
|
|
8
|
+
# entry — the rule locale files and database records share.
|
|
9
|
+
#
|
|
10
|
+
# An entry is the shape StateStore writes:
|
|
11
|
+
# { "source_hash" => …, "target_hash" => …, "manual" => true }
|
|
12
|
+
#
|
|
13
|
+
# Values may be a String or a nested Hash/Array of strings. A String hashes
|
|
14
|
+
# raw (exactly like StateStore, so existing state files stay valid); nested
|
|
15
|
+
# values hash over a canonical JSON form with sorted keys, so a record can
|
|
16
|
+
# track one entry per field or one per whole value.
|
|
17
|
+
class Drift
|
|
18
|
+
def initialize(hasher: StateStore.method(:hash))
|
|
19
|
+
@hasher = hasher
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def digest(value)
|
|
23
|
+
case value
|
|
24
|
+
when Hash, Array then @hasher.call(JSON.generate(canonical(value)))
|
|
25
|
+
else @hasher.call(value.to_s)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# A new entry for a target translated (or written by hand) from +source+.
|
|
30
|
+
def record(source:, target:, manual: false)
|
|
31
|
+
entry = { "source_hash" => digest(source), "target_hash" => digest(target) }
|
|
32
|
+
entry["manual"] = true if manual
|
|
33
|
+
entry
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# :missing (blank target), :outdated (source changed since the entry was
|
|
37
|
+
# recorded) or :current. The manual flag is not considered here; #due?
|
|
38
|
+
# applies it.
|
|
39
|
+
def status(source:, target:, entry:)
|
|
40
|
+
return :missing if blank?(target)
|
|
41
|
+
return :outdated if outdated?(source:, entry:)
|
|
42
|
+
|
|
43
|
+
:current
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The source changed since +entry+ recorded it. An untracked value (no
|
|
47
|
+
# entry or no source_hash) is not outdated.
|
|
48
|
+
def outdated?(source:, entry:)
|
|
49
|
+
stored = entry.is_a?(Hash) ? entry["source_hash"] : nil
|
|
50
|
+
!stored.nil? && stored != digest(source)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# A person wrote the target: it must not be machine-overwritten.
|
|
54
|
+
def manual?(entry) = entry.is_a?(Hash) && entry["manual"] ? true : false
|
|
55
|
+
|
|
56
|
+
# The target changed since it was recorded and is not yet flagged manual.
|
|
57
|
+
def hand_edited?(target:, entry:)
|
|
58
|
+
return false unless entry.is_a?(Hash) && !manual?(entry)
|
|
59
|
+
|
|
60
|
+
stored = entry["target_hash"]
|
|
61
|
+
!stored.nil? && stored != digest(target)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Whether the target should be (re)translated: missing or outdated, every
|
|
65
|
+
# value with +force+, and never a manual one unless +include_manual+.
|
|
66
|
+
def due?(source:, target:, entry:, force: false, include_manual: false)
|
|
67
|
+
return false if blank?(source)
|
|
68
|
+
return false if manual?(entry) && !include_manual
|
|
69
|
+
return true if force
|
|
70
|
+
|
|
71
|
+
status(source:, target:, entry:) != :current
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Empty, whitespace-only, or a Hash/Array with only blank leaves.
|
|
75
|
+
def blank?(value)
|
|
76
|
+
case value
|
|
77
|
+
when Hash then value.values.all? { |item| blank?(item) }
|
|
78
|
+
when Array then value.all? { |item| blank?(item) }
|
|
79
|
+
else value.to_s.strip.empty?
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def canonical(value)
|
|
86
|
+
case value
|
|
87
|
+
when Hash then value.map { |key, item| [key.to_s, canonical(item)] }.sort_by(&:first).to_h
|
|
88
|
+
when Array then value.map { |item| canonical(item) }
|
|
89
|
+
else value
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -24,10 +24,11 @@ module Locallingo
|
|
|
24
24
|
def extract_object(content)
|
|
25
25
|
text = content.to_s.strip
|
|
26
26
|
|
|
27
|
+
fenced = fenced_block(text)
|
|
27
28
|
parsed = try_parse(text) ||
|
|
28
|
-
try_parse(
|
|
29
|
+
try_parse(fenced) ||
|
|
29
30
|
first_balanced_object(text) ||
|
|
30
|
-
|
|
31
|
+
raise_parse_error(text, fenced)
|
|
31
32
|
|
|
32
33
|
# Both callers treat the result as a key->value Hash, so reject a
|
|
33
34
|
# top-level array (or any non-object) here with a clear contract error
|
|
@@ -46,6 +47,17 @@ module Locallingo
|
|
|
46
47
|
nil
|
|
47
48
|
end
|
|
48
49
|
|
|
50
|
+
# Raise the most useful parse error. A complete fence that fails to parse
|
|
51
|
+
# holds the real fault (e.g. an unescaped quote inside a value); parsing the
|
|
52
|
+
# raw text would only report the fence itself at line 1.
|
|
53
|
+
def raise_parse_error(text, fenced)
|
|
54
|
+
JSON.parse(fenced || text)
|
|
55
|
+
rescue JSON::ParserError => e
|
|
56
|
+
raise e unless fenced
|
|
57
|
+
|
|
58
|
+
raise JSON::ParserError, "#{e.message} (inside the fenced block)"
|
|
59
|
+
end
|
|
60
|
+
|
|
49
61
|
def fenced_block(text)
|
|
50
62
|
text[/```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```/m, 1]
|
|
51
63
|
end
|
data/lib/locallingo/manager.rb
CHANGED
|
@@ -8,7 +8,10 @@ require "fileutils"
|
|
|
8
8
|
require_relative "configuration"
|
|
9
9
|
require_relative "key_flattener"
|
|
10
10
|
require_relative "state_store"
|
|
11
|
+
require_relative "drift"
|
|
11
12
|
require_relative "providers/ruby_llm"
|
|
13
|
+
require_relative "batch_translator"
|
|
14
|
+
require_relative "prompt"
|
|
12
15
|
require_relative "validators/missing"
|
|
13
16
|
require_relative "validators/outdated"
|
|
14
17
|
require_relative "validators/duplicate_values"
|
|
@@ -25,9 +28,9 @@ module Locallingo
|
|
|
25
28
|
# Everything app-specific (locales, provider/model, prompt context/glossary,
|
|
26
29
|
# per-language guides, which validators run) comes from the Configuration.
|
|
27
30
|
class Manager
|
|
28
|
-
MAX_RETRIES =
|
|
29
|
-
MAX_MISSING_RETRIES =
|
|
30
|
-
BASE_SLEEP_DURATION =
|
|
31
|
+
MAX_RETRIES = BatchTranslator::MAX_RETRIES
|
|
32
|
+
MAX_MISSING_RETRIES = BatchTranslator::MAX_MISSING_RETRIES
|
|
33
|
+
BASE_SLEEP_DURATION = BatchTranslator::BASE_SLEEP_DURATION
|
|
31
34
|
|
|
32
35
|
attr_reader :config, :dry_run, :verbose, :logger, :cli_name
|
|
33
36
|
|
|
@@ -38,6 +41,7 @@ module Locallingo
|
|
|
38
41
|
@verbose = verbose
|
|
39
42
|
@cli_name = cli_name
|
|
40
43
|
@state = StateStore.new(@config.state_dir)
|
|
44
|
+
@drift = Drift.new
|
|
41
45
|
@provider = Providers::RubyLLM.new(provider: @config.provider)
|
|
42
46
|
@logger = logger || build_logger
|
|
43
47
|
end
|
|
@@ -69,7 +73,9 @@ module Locallingo
|
|
|
69
73
|
violations = []
|
|
70
74
|
source = load_source_translations
|
|
71
75
|
|
|
72
|
-
|
|
76
|
+
if config.validator_enabled?(:duplicate_values)
|
|
77
|
+
violations.concat(Validators::DuplicateValues.new.call(source:, locale: config.source_locale))
|
|
78
|
+
end
|
|
73
79
|
|
|
74
80
|
config.target_locales.each do |locale|
|
|
75
81
|
target = load_locale_translations(locale)
|
|
@@ -87,14 +93,20 @@ module Locallingo
|
|
|
87
93
|
violations
|
|
88
94
|
end
|
|
89
95
|
|
|
90
|
-
# Translate missing/changed keys for one or all target locales.
|
|
91
|
-
|
|
96
|
+
# Translate missing/changed keys for one or all target locales. Returns
|
|
97
|
+
# `{ locale => failed_keys }`; keys that failed are not written.
|
|
98
|
+
#
|
|
99
|
+
# Manual (hand-edited) keys are skipped by `force` and `force_keys`;
|
|
100
|
+
# `include_manual` lets the named `force_keys` overwrite them anyway.
|
|
101
|
+
def translate!(locale: nil, force: false, force_keys: [], include_manual: false)
|
|
92
102
|
@provider.ensure_credentials!
|
|
93
103
|
|
|
94
104
|
locales_to_process = locale ? [locale] : config.target_locales
|
|
95
105
|
source = load_source_translations
|
|
96
106
|
|
|
97
|
-
locales_to_process.
|
|
107
|
+
locales_to_process.to_h do |target_locale|
|
|
108
|
+
[target_locale, translate_locale(source, target_locale, force:, force_keys:, include_manual:)]
|
|
109
|
+
end
|
|
98
110
|
end
|
|
99
111
|
|
|
100
112
|
# Mark hand-edited target values as intentional (source_hash + target_hash +
|
|
@@ -118,11 +130,7 @@ module Locallingo
|
|
|
118
130
|
|
|
119
131
|
plans.each_with_object({}) do |(target_locale, target, locale_state, accepted), results|
|
|
120
132
|
accepted.each do |key|
|
|
121
|
-
locale_state[key] =
|
|
122
|
-
"source_hash" => @state.hash(source[key]),
|
|
123
|
-
"target_hash" => @state.hash(target[key]),
|
|
124
|
-
"manual" => true
|
|
125
|
-
}
|
|
133
|
+
locale_state[key] = @drift.record(source: source[key], target: target[key], manual: true)
|
|
126
134
|
end
|
|
127
135
|
@state.save(target_locale, locale_state) unless dry_run
|
|
128
136
|
results[target_locale] = accepted
|
|
@@ -168,6 +176,13 @@ module Locallingo
|
|
|
168
176
|
def missing_validator = @missing_validator ||= Validators::Missing.new(cli_name:)
|
|
169
177
|
def outdated_validator = @outdated_validator ||= Validators::Outdated.new(cli_name:)
|
|
170
178
|
|
|
179
|
+
def batch_translator
|
|
180
|
+
@batch_translator ||= BatchTranslator.new(
|
|
181
|
+
provider: @provider, model: config.translate_model, batch_size: config.batch_size,
|
|
182
|
+
max_batch_chars: config.max_batch_chars, pause: config.pause, log: method(:log)
|
|
183
|
+
)
|
|
184
|
+
end
|
|
185
|
+
|
|
171
186
|
def sync_locale_state(source, locale)
|
|
172
187
|
target = load_locale_translations(locale)
|
|
173
188
|
locale_state = @state.load(locale)
|
|
@@ -193,10 +208,7 @@ module Locallingo
|
|
|
193
208
|
return target.keys.select { |key| source.key?(key) } if all
|
|
194
209
|
|
|
195
210
|
target.keys.select do |key|
|
|
196
|
-
entry
|
|
197
|
-
next false unless source.key?(key) && entry.is_a?(Hash) && !entry["manual"]
|
|
198
|
-
|
|
199
|
-
entry["target_hash"] && entry["target_hash"] != @state.hash(target[key])
|
|
211
|
+
source.key?(key) && @drift.hand_edited?(target: target[key], entry: locale_state[key])
|
|
200
212
|
end
|
|
201
213
|
end
|
|
202
214
|
|
|
@@ -210,21 +222,30 @@ module Locallingo
|
|
|
210
222
|
raise Error, "accept-edits: key(s) not found in any target locale: #{missing.join(", ")}"
|
|
211
223
|
end
|
|
212
224
|
|
|
213
|
-
def translate_locale(source, target_locale, force:, force_keys:)
|
|
225
|
+
def translate_locale(source, target_locale, force:, force_keys:, include_manual:)
|
|
214
226
|
log("Processing #{target_locale}...")
|
|
215
227
|
|
|
216
228
|
target = load_locale_translations(target_locale)
|
|
217
229
|
locale_state = @state.load(target_locale)
|
|
218
230
|
exceptions = load_exceptions(target_locale)
|
|
219
231
|
|
|
232
|
+
if force_keys.any? && !force
|
|
233
|
+
force_keys = resolve_force_keys(source, force_keys, locale_state, include_manual:, locale: target_locale)
|
|
234
|
+
# Every named key was manual: translate nothing rather than falling
|
|
235
|
+
# back to the missing/outdated keys nobody asked for.
|
|
236
|
+
if force_keys.empty?
|
|
237
|
+
log(" No keys to translate for #{target_locale} (every --force-key is manual)")
|
|
238
|
+
return []
|
|
239
|
+
end
|
|
240
|
+
end
|
|
220
241
|
keys = determine_keys_to_translate(source, target, locale_state, force:, force_keys:, exceptions:)
|
|
221
242
|
if keys.empty?
|
|
222
243
|
log(" No keys to translate for #{target_locale}")
|
|
223
|
-
return
|
|
244
|
+
return []
|
|
224
245
|
end
|
|
225
246
|
|
|
226
247
|
log(" Translating #{keys.size} keys...")
|
|
227
|
-
translated, failed =
|
|
248
|
+
translated, failed = batch_translator.call(source, keys, instructions: translation_prompt(target_locale))
|
|
228
249
|
|
|
229
250
|
successful = translated.except(*failed)
|
|
230
251
|
unless dry_run
|
|
@@ -234,103 +255,13 @@ module Locallingo
|
|
|
234
255
|
end
|
|
235
256
|
|
|
236
257
|
log(" Completed #{target_locale}: #{successful.size} translated, #{failed.size} failed")
|
|
258
|
+
failed
|
|
237
259
|
end
|
|
238
260
|
|
|
239
|
-
def
|
|
240
|
-
translated, failed = translate_keys(source, keys, target_locale)
|
|
241
|
-
|
|
242
|
-
round = 0
|
|
243
|
-
while failed.any? && round < MAX_MISSING_RETRIES
|
|
244
|
-
round += 1
|
|
245
|
-
log(" Retry round #{round}: #{failed.size} keys remaining...")
|
|
246
|
-
sleep(BASE_SLEEP_DURATION * (2**round))
|
|
247
|
-
retried, failed = translate_keys(source, failed, target_locale)
|
|
248
|
-
translated.merge!(retried)
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
if failed.any?
|
|
252
|
-
log(" WARNING: #{failed.size} keys failed after all retries:", level: :warn)
|
|
253
|
-
failed.each { |key| log(" - #{key}", level: :warn) }
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
[translated, failed]
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
def translate_keys(source, keys, target_locale)
|
|
260
|
-
translations = {}
|
|
261
|
-
failed = []
|
|
262
|
-
|
|
263
|
-
keys.each_slice(config.batch_size) do |batch|
|
|
264
|
-
result = translate_batch(batch.to_h { |key| [key, source[key]] }, target_locale)
|
|
265
|
-
batch.each do |key|
|
|
266
|
-
if result.key?(key) && !result[key].to_s.empty?
|
|
267
|
-
translations[key] = result[key]
|
|
268
|
-
else
|
|
269
|
-
failed << key
|
|
270
|
-
end
|
|
271
|
-
end
|
|
272
|
-
sleep(BASE_SLEEP_DURATION)
|
|
273
|
-
end
|
|
274
|
-
|
|
275
|
-
[translations, failed]
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
def translate_batch(payload, target_locale)
|
|
279
|
-
return {} if payload.empty?
|
|
280
|
-
|
|
281
|
-
retries = 0
|
|
282
|
-
begin
|
|
283
|
-
result = @provider.chat(
|
|
284
|
-
model: config.translate_model,
|
|
285
|
-
instructions: translation_prompt(target_locale),
|
|
286
|
-
payload:
|
|
287
|
-
)
|
|
288
|
-
log(" Batch translated: #{result.keys.size}/#{payload.keys.size} keys")
|
|
289
|
-
result
|
|
290
|
-
rescue StandardError => e
|
|
291
|
-
retries += 1
|
|
292
|
-
if retries < MAX_RETRIES
|
|
293
|
-
sleep_duration = BASE_SLEEP_DURATION * (2**retries)
|
|
294
|
-
log(" Batch failed (attempt #{retries}), retrying in #{sleep_duration}s: #{e.message}", level: :warn)
|
|
295
|
-
sleep(sleep_duration)
|
|
296
|
-
retry
|
|
297
|
-
end
|
|
298
|
-
log(" Translation batch failed after #{MAX_RETRIES} retries: #{e.message}", level: :error)
|
|
299
|
-
{}
|
|
300
|
-
end
|
|
301
|
-
end
|
|
302
|
-
|
|
303
|
-
def translation_prompt(locale)
|
|
304
|
-
<<~PROMPT
|
|
305
|
-
Translate the following texts from #{config.language_name(config.source_locale)} to #{config.language_name(locale)}.
|
|
306
|
-
This is for #{config.context}.
|
|
307
|
-
|
|
308
|
-
## General Rules
|
|
309
|
-
- Preserve placeholders like #{config.placeholder_style} exactly as they appear
|
|
310
|
-
- Preserve HTML tags if present
|
|
311
|
-
- Use formal business language
|
|
312
|
-
- Keep translations concise - UI space is limited
|
|
313
|
-
#{glossary_section}#{language_guide_section(locale)}
|
|
314
|
-
Return ONLY a raw JSON object mapping each input key to its translation,
|
|
315
|
-
with no surrounding prose and no markdown code fences:
|
|
316
|
-
{"key1": "translated1", "key2": "translated2"}
|
|
317
|
-
PROMPT
|
|
318
|
-
end
|
|
319
|
-
|
|
320
|
-
def glossary_section
|
|
321
|
-
return "" if config.glossary.empty?
|
|
322
|
-
|
|
323
|
-
lines = config.glossary.map { |term, meaning| " - \"#{term}\" = #{meaning}" }
|
|
324
|
-
"\n## Terminology\n#{lines.join("\n")}\n"
|
|
325
|
-
end
|
|
326
|
-
|
|
327
|
-
def language_guide_section(locale)
|
|
328
|
-
guide = config.language_guide(locale)
|
|
329
|
-
guide.empty? ? "" : "\n#{guide}\n"
|
|
330
|
-
end
|
|
261
|
+
def translation_prompt(locale) = Prompt.translation(config:, from: config.source_locale, to: locale)
|
|
331
262
|
|
|
332
263
|
def determine_keys_to_translate(source, target, locale_state, force:, force_keys:, exceptions:)
|
|
333
|
-
manual_keys = locale_state.select { |
|
|
264
|
+
manual_keys = locale_state.select { |_key, entry| @drift.manual?(entry) }.keys.to_set
|
|
334
265
|
return source.keys - manual_keys.to_a if force
|
|
335
266
|
return force_keys & source.keys if force_keys.any?
|
|
336
267
|
|
|
@@ -340,19 +271,35 @@ module Locallingo
|
|
|
340
271
|
(missing + outdated).uniq
|
|
341
272
|
end
|
|
342
273
|
|
|
274
|
+
# Drops manual keys from the named force_keys (with a loud warning) unless
|
|
275
|
+
# the caller opted in to overwriting them.
|
|
276
|
+
def resolve_force_keys(source, force_keys, locale_state, include_manual:, locale:)
|
|
277
|
+
manual = (force_keys & source.keys).select { |key| @drift.manual?(locale_state[key]) }
|
|
278
|
+
return force_keys if manual.empty?
|
|
279
|
+
|
|
280
|
+
if include_manual
|
|
281
|
+
warn_manual_keys(manual, locale, dry_run ? "would overwrite" : "overwriting")
|
|
282
|
+
force_keys
|
|
283
|
+
else
|
|
284
|
+
warn_manual_keys(manual, locale, dry_run ? "would skip" : "skipping", hint: true)
|
|
285
|
+
force_keys - manual
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def warn_manual_keys(keys, locale, verb, hint: false)
|
|
290
|
+
keys.each { |key| warn("#{verb} manual (hand-edited) key #{key} for #{locale}") }
|
|
291
|
+
warn("Pass --include-manual to overwrite manual keys named via --force-key.") if hint
|
|
292
|
+
end
|
|
293
|
+
|
|
343
294
|
def update_locale_state(source, locale_state, translations)
|
|
344
295
|
translations.each_key do |key|
|
|
345
296
|
next unless source[key]
|
|
346
297
|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
# A force-keyed retranslation may overwrite a manual value on explicit
|
|
353
|
-
# request, but the protection flag itself must survive.
|
|
354
|
-
entry["manual"] = true if existing.is_a?(Hash) && existing["manual"]
|
|
355
|
-
locale_state[key] = entry
|
|
298
|
+
# A force-keyed retranslation overwrites a manual value only with
|
|
299
|
+
# include_manual, and even then the protection flag must survive.
|
|
300
|
+
locale_state[key] = @drift.record(
|
|
301
|
+
source: source[key], target: translations[key], manual: @drift.manual?(locale_state[key])
|
|
302
|
+
)
|
|
356
303
|
end
|
|
357
304
|
end
|
|
358
305
|
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Locallingo
|
|
4
|
+
# Builds the translation system prompt: idiomatic-translation and formatting
|
|
5
|
+
# rules, the configured glossary and per-language guide, plus optional caller
|
|
6
|
+
# instructions.
|
|
7
|
+
#
|
|
8
|
+
# Styles:
|
|
9
|
+
# - `:ui` — short interface strings (the CLI's prompt, pinned by a spec)
|
|
10
|
+
# - `:prose` — long-form content: Markdown structure is kept and there is no
|
|
11
|
+
# brevity rule
|
|
12
|
+
module Prompt
|
|
13
|
+
STYLES = %i[ui prose].freeze
|
|
14
|
+
|
|
15
|
+
BREVITY_RULE = "- Keep translations concise - UI space is limited\n"
|
|
16
|
+
|
|
17
|
+
MARKDOWN_RULES = <<~RULES
|
|
18
|
+
- Keep the Markdown structure: headings, lists, emphasis, links and line breaks stay where they are
|
|
19
|
+
- Translate link text, never the link target (URL)
|
|
20
|
+
- Never merge or split values: every key gets exactly one translated value
|
|
21
|
+
RULES
|
|
22
|
+
|
|
23
|
+
module_function
|
|
24
|
+
|
|
25
|
+
def translation(config:, from:, to:, style: :ui, instructions: nil)
|
|
26
|
+
raise Error, "Unknown prompt style #{style.inspect} (expected one of #{STYLES.join(", ")})" unless
|
|
27
|
+
STYLES.include?(style.to_s.to_sym)
|
|
28
|
+
|
|
29
|
+
language = config.language_name(to)
|
|
30
|
+
<<~PROMPT
|
|
31
|
+
Translate the following texts from #{config.language_name(from)} into natural, idiomatic #{language}.
|
|
32
|
+
This is for #{config.context}.
|
|
33
|
+
|
|
34
|
+
## Idiomatic Translation
|
|
35
|
+
- Translate the meaning, not word for word: write what a native #{language} speaker would actually say in this situation
|
|
36
|
+
- Use the phrasing, word order and UI conventions that are standard in #{language} software, even when they differ from the source
|
|
37
|
+
- Replace source-language idioms and set phrases with their natural #{language} equivalent instead of translating them literally
|
|
38
|
+
- Avoid calques and anglicisms when an established #{language} expression exists
|
|
39
|
+
- Never add, drop or change information to make a sentence sound natural
|
|
40
|
+
- Terminology and any language guide below take precedence over these style rules
|
|
41
|
+
|
|
42
|
+
## General Rules
|
|
43
|
+
- Preserve placeholders like #{config.placeholder_style} exactly as they appear
|
|
44
|
+
- Preserve HTML tags if present
|
|
45
|
+
#{style.to_sym == :prose ? MARKDOWN_RULES : BREVITY_RULE}- Escape every double quote (") inside a translated value as \\" so the reply stays valid JSON
|
|
46
|
+
- Keep typographic quotes (“ ” „ ‚ « ») exactly as they appear in the source
|
|
47
|
+
#{glossary_section(config)}#{language_guide_section(config, to)}#{instructions_section(instructions)}
|
|
48
|
+
Return ONLY a raw JSON object mapping each input key to its translation,
|
|
49
|
+
with no surrounding prose and no markdown code fences:
|
|
50
|
+
{"key1": "translated1", "key2": "translated2"}
|
|
51
|
+
PROMPT
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def glossary_section(config)
|
|
55
|
+
return "" if config.glossary.empty?
|
|
56
|
+
|
|
57
|
+
lines = config.glossary.map { |term, meaning| " - \"#{term}\" = #{meaning}" }
|
|
58
|
+
"\n## Terminology\n#{lines.join("\n")}\n"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def language_guide_section(config, locale)
|
|
62
|
+
guide = config.language_guide(locale)
|
|
63
|
+
guide.empty? ? "" : "\n#{guide}\n"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def instructions_section(instructions)
|
|
67
|
+
text = instructions.to_s.strip
|
|
68
|
+
text.empty? ? "" : "\n## Additional Instructions\n#{text}\n"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private_class_method :glossary_section, :language_guide_section, :instructions_section
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -13,6 +13,16 @@ module Locallingo
|
|
|
13
13
|
# truth) so independent batches don't share conversation history. The
|
|
14
14
|
# response is parsed with the robust JsonExtraction extractor because not
|
|
15
15
|
# every provider guarantees fenceless JSON.
|
|
16
|
+
#
|
|
17
|
+
# The system prompt is marked as a prompt cache boundary: the translation
|
|
18
|
+
# prompt is built once per locale and reused by every batch, so providers
|
|
19
|
+
# that support caching (Anthropic, OpenAI-compatible) bill the repeated
|
|
20
|
+
# prefix at the cached rate.
|
|
21
|
+
#
|
|
22
|
+
# Given a `RubyLLM::Context` (`RubyLLM.context { |c| c.anthropic_api_key = … }`),
|
|
23
|
+
# every chat runs on that context's own configuration: the global
|
|
24
|
+
# `RubyLLM.config` is never written, so translators with different keys
|
|
25
|
+
# (one per tenant) can run concurrently.
|
|
16
26
|
class RubyLLM
|
|
17
27
|
# Maps a RubyLLM provider symbol to the ENV var whose presence indicates
|
|
18
28
|
# credentials are available, so we can fail fast with a clear message
|
|
@@ -25,10 +35,11 @@ module Locallingo
|
|
|
25
35
|
openrouter: "OPENROUTER_API_KEY"
|
|
26
36
|
}.freeze
|
|
27
37
|
|
|
28
|
-
attr_reader :provider
|
|
38
|
+
attr_reader :provider, :context
|
|
29
39
|
|
|
30
|
-
def initialize(provider:)
|
|
40
|
+
def initialize(provider:, context: nil)
|
|
31
41
|
@provider = provider.to_sym
|
|
42
|
+
@context = context
|
|
32
43
|
end
|
|
33
44
|
|
|
34
45
|
# True when a key for the configured provider is found in any source
|
|
@@ -45,6 +56,12 @@ module Locallingo
|
|
|
45
56
|
def ensure_credentials!
|
|
46
57
|
return if credentials?
|
|
47
58
|
|
|
59
|
+
if context
|
|
60
|
+
raise MissingCredentialsError,
|
|
61
|
+
"No credentials for provider #{provider.inspect}: the given RubyLLM context " \
|
|
62
|
+
"has no #{key_setting}."
|
|
63
|
+
end
|
|
64
|
+
|
|
48
65
|
raise MissingCredentialsError,
|
|
49
66
|
"No credentials for provider #{provider.inspect}. " \
|
|
50
67
|
"Set #{CREDENTIAL_ENV[provider]} in ENV, call " \
|
|
@@ -56,20 +73,21 @@ module Locallingo
|
|
|
56
73
|
# the model and return the parsed JSON object as a Hash.
|
|
57
74
|
def chat(model:, instructions:, payload:)
|
|
58
75
|
require "ruby_llm"
|
|
59
|
-
configure_credentials!
|
|
60
|
-
|
|
61
|
-
conversation = ::RubyLLM.chat(
|
|
62
|
-
model:,
|
|
63
|
-
provider:,
|
|
64
|
-
assume_model_exists: true
|
|
65
|
-
).with_instructions(instructions)
|
|
66
76
|
|
|
77
|
+
conversation = new_chat(model).with_instructions(instructions, cache_until_here: true)
|
|
67
78
|
response = conversation.ask(JSON.pretty_generate(payload))
|
|
68
79
|
JsonExtraction.extract_object(response.content)
|
|
69
80
|
end
|
|
70
81
|
|
|
71
82
|
private
|
|
72
83
|
|
|
84
|
+
def new_chat(model)
|
|
85
|
+
return context.chat(model:, provider:, assume_model_exists: true) if context
|
|
86
|
+
|
|
87
|
+
configure_credentials!
|
|
88
|
+
::RubyLLM.chat(model:, provider:, assume_model_exists: true)
|
|
89
|
+
end
|
|
90
|
+
|
|
73
91
|
# RubyLLM does not read provider API keys from ENV on its own, so a
|
|
74
92
|
# standalone CLI run (no Rails initializer to call RubyLLM.configure)
|
|
75
93
|
# would raise "Missing configuration for <provider>". Push the resolved
|
|
@@ -93,9 +111,18 @@ module Locallingo
|
|
|
93
111
|
# Key resolution across sources, in precedence order. Used by
|
|
94
112
|
# #credentials? as a fail-fast check before any network call.
|
|
95
113
|
def resolved_api_key
|
|
114
|
+
return context_api_key if context
|
|
115
|
+
|
|
96
116
|
settings_api_key || host_configured_api_key || env_api_key
|
|
97
117
|
end
|
|
98
118
|
|
|
119
|
+
# With a context, its own config is the only source: the chat never
|
|
120
|
+
# reads the settings, the global config or ENV.
|
|
121
|
+
def context_api_key
|
|
122
|
+
config = context.config
|
|
123
|
+
config.respond_to?(key_setting) ? presence(config.public_send(key_setting)) : nil
|
|
124
|
+
end
|
|
125
|
+
|
|
99
126
|
def settings_api_key
|
|
100
127
|
Locallingo.settings.api_key_for(provider)
|
|
101
128
|
end
|
|
@@ -31,13 +31,15 @@ module Locallingo
|
|
|
31
31
|
@terminology = Quality::Terminology.new(@config.terminology_setting, base_path: @config.base_path)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
# Check
|
|
35
|
-
|
|
34
|
+
# Check the translations for a locale: all of them, those under +prefix+
|
|
35
|
+
# (e.g. "yoga."), or exactly the named +keys+. The AI pass reviews a sample
|
|
36
|
+
# of up to `quality.sample_size` keys, or every named key.
|
|
37
|
+
def check(locale: nil, use_ai: false, prefix: nil, keys: nil)
|
|
36
38
|
locale ||= config.source_locale
|
|
37
|
-
translations =
|
|
39
|
+
translations = scoped_translations(locale, prefix:, keys:)
|
|
38
40
|
|
|
39
41
|
suggestions = translations.flat_map { |key, text| check_text(key, text, locale) }
|
|
40
|
-
suggestions.concat(ai_sample(translations, locale)) if use_ai
|
|
42
|
+
suggestions.concat(keys ? suggest_improvements(translations, locale) : ai_sample(translations, locale)) if use_ai
|
|
41
43
|
suggestions
|
|
42
44
|
end
|
|
43
45
|
|
|
@@ -116,8 +118,26 @@ module Locallingo
|
|
|
116
118
|
}
|
|
117
119
|
end
|
|
118
120
|
|
|
121
|
+
def scoped_translations(locale, prefix:, keys:)
|
|
122
|
+
translations = load_locale_translations(locale)
|
|
123
|
+
return slice_keys(translations, keys, locale) if keys
|
|
124
|
+
return translations unless prefix
|
|
125
|
+
|
|
126
|
+
scoped = translations.select { |key, _| key.start_with?(prefix) }
|
|
127
|
+
raise Error, "No keys match prefix #{prefix.inspect} in #{locale}" if scoped.empty?
|
|
128
|
+
|
|
129
|
+
scoped
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def slice_keys(translations, keys, locale)
|
|
133
|
+
missing = keys - translations.keys
|
|
134
|
+
raise Error, "quality: key(s) not found in #{locale}: #{missing.join(", ")}" if missing.any?
|
|
135
|
+
|
|
136
|
+
translations.slice(*keys)
|
|
137
|
+
end
|
|
138
|
+
|
|
119
139
|
def ai_sample(translations, locale)
|
|
120
|
-
sample_size = [translations.size,
|
|
140
|
+
sample_size = [translations.size, config.quality_sample_size].min
|
|
121
141
|
sample = translations.to_a.sample(sample_size).to_h
|
|
122
142
|
suggest_improvements(sample, locale)
|
|
123
143
|
end
|
|
@@ -153,7 +173,7 @@ module Locallingo
|
|
|
153
173
|
<<~PROMPT
|
|
154
174
|
Review these UI translations for #{config.context}.
|
|
155
175
|
Suggest improvements for clarity, professionalism, and user-friendliness.
|
|
156
|
-
#{glossary_section}
|
|
176
|
+
#{glossary_section}#{language_guide_section(locale)}
|
|
157
177
|
For each translation that needs improvement, provide:
|
|
158
178
|
1. The issue (brief)
|
|
159
179
|
2. Suggested improvement
|
|
@@ -174,6 +194,11 @@ module Locallingo
|
|
|
174
194
|
"\n#{config.context} terminology:\n#{lines.join("\n")}\n"
|
|
175
195
|
end
|
|
176
196
|
|
|
197
|
+
def language_guide_section(locale)
|
|
198
|
+
guide = config.language_guide(locale)
|
|
199
|
+
guide.empty? ? "" : "\n#{guide}\n"
|
|
200
|
+
end
|
|
201
|
+
|
|
177
202
|
def load_locale_translations(locale)
|
|
178
203
|
translations = {}
|
|
179
204
|
patterns = [
|