locallingo 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -0
- data/README.md +106 -1
- data/config/locallingo.default.yml +6 -0
- data/lib/locallingo/active_record.rb +167 -0
- data/lib/locallingo/batch_translator.rb +26 -3
- data/lib/locallingo/configuration.rb +40 -18
- data/lib/locallingo/content_leaves.rb +40 -0
- data/lib/locallingo/drift.rb +93 -0
- data/lib/locallingo/manager.rb +16 -59
- data/lib/locallingo/prompt.rb +73 -0
- data/lib/locallingo/providers/ruby_llm.rb +43 -26
- data/lib/locallingo/quality/static_rules.rb +6 -5
- data/lib/locallingo/quality/terminology.rb +1 -1
- data/lib/locallingo/record_translator.rb +94 -0
- data/lib/locallingo/rubocop.rb +1 -1
- data/lib/locallingo/settings.rb +12 -1
- data/lib/locallingo/translator.rb +70 -0
- data/lib/locallingo/validators/manual_edits.rb +4 -8
- data/lib/locallingo/validators/outdated.rb +5 -5
- data/lib/locallingo/version.rb +1 -1
- data/lib/locallingo.rb +50 -6
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 47b5cfaffa1567e536de600f6f963538606fc386a89b61e90ad9cc4dd8327132
|
|
4
|
+
data.tar.gz: 767fa1b9c49c66a0b5b5e1bd6140edbca50edade201f415c9bfde2c6f0448206
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fad2120d8032a4a019dda5f6c82cd2f597de9dd79f9bb4eab1b9026530c859316c232314325398f51e12d61fc043c95882960969d5be325a80e6676463b55862
|
|
7
|
+
data.tar.gz: ff18516ac297ff0612111d0326552e6e771a6e0299854c598f3ea5645e9baaf722dd80e2571fc68186f3225af3eb35a69dc496107904feedd2cfc9d7356b1e8a
|
data/CHANGELOG.md
CHANGED
|
@@ -4,9 +4,75 @@ All notable changes to this project are documented here. The format is based on
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
5
|
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [0.6.0]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- `Locallingo::Configuration.parse(text, root_path:, package:)` resolves
|
|
11
|
+
config from YAML text (ERB included) instead of `.locallingo.yml`, merged
|
|
12
|
+
over the shipped defaults the same way. (#35)
|
|
13
|
+
- Concurrency conformance, development only (no new runtime dependency): CI
|
|
14
|
+
runs the suite on Ruby 4.0 too, and every pull request runs
|
|
15
|
+
conformance-kit 0.4's Conformance workflow: thread and fiber contention
|
|
16
|
+
examples, a Ractor canary of the pure core on Ruby 4.0, and a strict audit
|
|
17
|
+
of the gem's shared state. (#35)
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
- Settings are frozen after boot, with nothing to call: the default settings
|
|
21
|
+
are frozen at load, and `Locallingo.configure` (a Rails initializer, or the
|
|
22
|
+
CLI's `.locallingo.rb`) publishes them frozen, in one assignment, so a
|
|
23
|
+
concurrent reader sees the old settings or the new, never half of them.
|
|
24
|
+
Concurrent `configure` calls run one at a time, so neither drops the
|
|
25
|
+
other's keys. Change them by calling `configure` again: **writing to
|
|
26
|
+
`Locallingo.settings` directly now raises `FrozenError`.** String keys are
|
|
27
|
+
stored as frozen plain String copies (a String subclass or its ivars are not
|
|
28
|
+
kept), so the settings are Ractor-shareable. A callable
|
|
29
|
+
key is kept as it is and never makes `configure` raise, even when it closes
|
|
30
|
+
over app state (a logger, a vault client); it keeps the settings readable
|
|
31
|
+
on the main Ractor only. (#35)
|
|
32
|
+
- `translates_content` freezes its declaration at boot: the column and state
|
|
33
|
+
names and a String or Array `source_locale:`/`target_locales:` (as frozen
|
|
34
|
+
copies; the objects passed in are not frozen), and the drift. **Mutating them
|
|
35
|
+
through `locallingo_translatable` now raises `FrozenError`.** Callables
|
|
36
|
+
(`source_locale:`, `target_locales:`, `translatable:`, `hasher:`) are kept
|
|
37
|
+
as they are and never make the declaration raise. (#35)
|
|
38
|
+
|
|
39
|
+
### Fixed
|
|
40
|
+
- Without a `RubyLLM::Context`, a translation or quality-review call no
|
|
41
|
+
longer writes the resolved key into the process-global `RubyLLM.config`;
|
|
42
|
+
each call runs on a context of its own. Two translators with different keys
|
|
43
|
+
in one process could race, and one chat with the other's key. (#35)
|
|
44
|
+
- `after_translate` hooks run via `system(command, chdir:)` instead of inside
|
|
45
|
+
`Dir.chdir`, which moved the working directory of the whole process while a
|
|
46
|
+
hook ran. (#35)
|
|
47
|
+
- The static quality checks and `Configuration` (its ERB included) run inside
|
|
48
|
+
a non-main Ractor on Ruby 4.0: nested constants are frozen, and ERB no
|
|
49
|
+
longer evaluates on `TOPLEVEL_BINDING` (templates still resolve constants
|
|
50
|
+
from the top level, as the app sees them). (#35)
|
|
51
|
+
|
|
7
52
|
## [Unreleased]
|
|
8
53
|
|
|
9
54
|
### Added
|
|
55
|
+
- `Locallingo::Translator#translate(texts, from:, to:, style:, instructions:)`
|
|
56
|
+
translates a plain `{ key => text }` hash with the configured prompt,
|
|
57
|
+
glossary, language guide, provider and model, and returns `translations` and
|
|
58
|
+
`failed`. (#33)
|
|
59
|
+
- `Locallingo::Prompt.translation`: the translation prompt, public. `style:
|
|
60
|
+
:prose` keeps Markdown structure and drops the UI brevity rule;
|
|
61
|
+
`instructions:` is appended after the language guide. The CLI's `:ui`
|
|
62
|
+
prompt is unchanged. (#33)
|
|
63
|
+
- `Providers::RubyLLM.new(provider:, context:)` and `Translator.new(context:)`
|
|
64
|
+
take a `RubyLLM::Context`; with one, the global `RubyLLM.config` is never
|
|
65
|
+
written. (#33)
|
|
66
|
+
- `translate.max_batch_chars` limits the source characters per batch (a single
|
|
67
|
+
value over the limit is sent alone, never split); `translate.pause` sets the
|
|
68
|
+
sleep after each batch (default `1.0`). (#33)
|
|
69
|
+
- `Locallingo::Drift`: missing / outdated / manual / hand-edited decisions as a
|
|
70
|
+
value over the state entry shape, for strings and nested values, with a
|
|
71
|
+
pluggable hasher. `Manager` and the validators use it; behaviour is
|
|
72
|
+
unchanged. (#33)
|
|
73
|
+
- `require "locallingo/active_record"` adds `Locallingo::Translatable`
|
|
74
|
+
(`translates_content`), and `Locallingo::RecordTranslator` translates many
|
|
75
|
+
records in one batched call, writing each under its row lock. (#33)
|
|
10
76
|
- `lingo quality --prefix yoga.` checks only the keys under a prefix, in both
|
|
11
77
|
the static checks and the AI pass. A prefix that matches nothing exits `1`. (#10, #29)
|
|
12
78
|
- `lingo quality --key a.b` (repeatable) checks exactly the named keys, and the
|
data/README.md
CHANGED
|
@@ -46,7 +46,12 @@ end
|
|
|
46
46
|
Put that in a `.locallingo.rb` file next to `.locallingo.yml` and the `lingo`
|
|
47
47
|
CLI loads it on start — no Rails boot required. A key set via
|
|
48
48
|
`Locallingo.configure` wins over one set through `RubyLLM.configure`, which
|
|
49
|
-
wins over ENV. Locallingo never stores keys itself.
|
|
49
|
+
wins over ENV. Locallingo never stores keys itself. `configure` publishes the
|
|
50
|
+
settings frozen: change them by calling it again, never by writing to
|
|
51
|
+
`Locallingo.settings`. Nothing else needs calling at boot. With String keys
|
|
52
|
+
the settings are Ractor-shareable; a callable key is kept as it is (resolved on
|
|
53
|
+
every use, so it may close over a logger or a vault client), which keeps the
|
|
54
|
+
settings readable on the main Ractor only.
|
|
50
55
|
|
|
51
56
|
## Configuration
|
|
52
57
|
|
|
@@ -146,6 +151,106 @@ Locallingo::Manager.new(config:).validate # => [violations]
|
|
|
146
151
|
Locallingo::QualityChecker.new(config:).check(locale: "en")
|
|
147
152
|
```
|
|
148
153
|
|
|
154
|
+
`Locallingo::Configuration.parse(yaml, root_path:, package:)` resolves the same
|
|
155
|
+
config from YAML text (ERB included) instead of reading `.locallingo.yml`. A
|
|
156
|
+
configuration loaded from the file reads it when it is built, never cached.
|
|
157
|
+
|
|
158
|
+
### Translating text that is not in locale files
|
|
159
|
+
|
|
160
|
+
`Locallingo::Translator` runs the same prompt (rules, glossary, language guide),
|
|
161
|
+
provider, model and batching over a plain `{ key => text }` hash:
|
|
162
|
+
|
|
163
|
+
```ruby
|
|
164
|
+
require "ruby_llm" # for RubyLLM.context (Bundler.require already loads it in Rails)
|
|
165
|
+
|
|
166
|
+
translator = Locallingo::Translator.new(
|
|
167
|
+
config: Locallingo.configuration(root_path: Dir.pwd),
|
|
168
|
+
context: RubyLLM.context { |c| c.anthropic_api_key = tenant_key }, # optional, per-tenant key
|
|
169
|
+
pause: 0 # no sleep between batches
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
result = translator.translate(
|
|
173
|
+
{ "title" => "Hello", "intro" => "Some **markdown**." },
|
|
174
|
+
from: "en", to: "de",
|
|
175
|
+
style: :prose, # :ui (default) or :prose (keeps Markdown, no brevity rule)
|
|
176
|
+
instructions: "Write in a warm, plain voice." # appended after the language guide
|
|
177
|
+
)
|
|
178
|
+
result.translations # => { "title" => "Hallo", "intro" => "Etwas **Markdown**." }
|
|
179
|
+
result.failed # => keys that failed after retries
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
- Every call runs on a `RubyLLM::Context`: the `context:` given, else one
|
|
183
|
+
built for that call from the resolved key. The global `RubyLLM.config` is
|
|
184
|
+
never written, so tenants can translate concurrently.
|
|
185
|
+
- `MissingCredentialsError` is raised before any call when no key is found.
|
|
186
|
+
- `translate.max_batch_chars` (or `max_batch_chars:`) limits the source
|
|
187
|
+
characters per batch (a single longer value is sent alone, not split); `translate.pause` (or `pause:`) is the sleep after each
|
|
188
|
+
batch. Retries of failed or omitted keys keep their own backoff.
|
|
189
|
+
- `Locallingo::Prompt.translation(config:, from:, to:, style:, instructions:)`
|
|
190
|
+
returns the prompt itself.
|
|
191
|
+
|
|
192
|
+
`Locallingo::Drift` is the missing / outdated / hand-edited rule over the state
|
|
193
|
+
entry shape (`source_hash`, `target_hash`, `manual`), for strings or nested
|
|
194
|
+
values:
|
|
195
|
+
|
|
196
|
+
```ruby
|
|
197
|
+
drift = Locallingo::Drift.new # hasher: any callable(String) -> String
|
|
198
|
+
drift.status(source:, target:, entry:) # => :missing | :outdated | :current
|
|
199
|
+
drift.due?(source:, target:, entry:, force: false, include_manual: false)
|
|
200
|
+
drift.hand_edited?(target:, entry:)
|
|
201
|
+
drift.record(source:, target:, manual: false) # => a new entry
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
### Translating records
|
|
205
|
+
|
|
206
|
+
`require "locallingo/active_record"` (ActiveRecord >= 7.1; nothing loads
|
|
207
|
+
ActiveRecord otherwise) adds a concern for a JSON(B) column keyed by locale
|
|
208
|
+
plus a JSON(B) state column:
|
|
209
|
+
|
|
210
|
+
```ruby
|
|
211
|
+
class Article < ApplicationRecord
|
|
212
|
+
include Locallingo::Translatable
|
|
213
|
+
|
|
214
|
+
translates_content :body, # { "en" => { … }, "de" => { … } }
|
|
215
|
+
state: :translation_state, # { "de" => { "source_hash", "target_hash", "manual" } }
|
|
216
|
+
source_locale: -> { "en" }, # String/Symbol, or a callable run on the record
|
|
217
|
+
target_locales: -> { %w[de sv] },
|
|
218
|
+
translatable: ->(path, value) { true } # optional filter on flattened String leaves
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
article.translation_status(:de) # => :missing | :outdated | :current
|
|
222
|
+
article.translation_manual?(:de)
|
|
223
|
+
article.translation_due?(:de, force: false)
|
|
224
|
+
article.record_translation(:de, manual: false)
|
|
225
|
+
|
|
226
|
+
Locallingo::RecordTranslator.new(translator:)
|
|
227
|
+
.call(Article.where(id: ids), to: "de", style: :prose)
|
|
228
|
+
# => { translated: 3, failed: { article_id => [paths] }, skipped: [ids] }
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
- Saving a record runs `mark_hand_edits`: a target locale a person changed is
|
|
232
|
+
flagged `manual` and never machine-overwritten (`include_manual: true`
|
|
233
|
+
overrides, keeping the flag). That includes a target locale saved for the
|
|
234
|
+
first time, e.g. a source copy used as a placeholder at `create!`: leave
|
|
235
|
+
untranslated locales out of the column instead. A cleared locale is not
|
|
236
|
+
flagged and is due again, unless it was already `manual`: the flag stays, so
|
|
237
|
+
retranslating it needs `include_manual: true`.
|
|
238
|
+
- `RecordTranslator` sends one batched translation per source locale across all
|
|
239
|
+
due records, then writes each record under `record.with_lock`. A record whose
|
|
240
|
+
source changed, or whose target became manual, while the model answered is
|
|
241
|
+
skipped (`skipped:`) and left for the next run, as is a due record with no
|
|
242
|
+
translatable text.
|
|
243
|
+
- Nested values keep their shape; leaves the `translatable:` filter rejects are
|
|
244
|
+
copied from the source. A record with any segment that failed after retries
|
|
245
|
+
is not written at all: its paths are reported in `failed:`, and it stays due
|
|
246
|
+
for the next run (your fallback to the source locale applies meanwhile).
|
|
247
|
+
- Writes use `update_columns`; pass `write: ->(record, attributes) { … }` to
|
|
248
|
+
bump a lock version or run validations.
|
|
249
|
+
- `translates_content` freezes the declaration as it is made, at boot: the
|
|
250
|
+
column and state names, a String or Array locale option (as a frozen copy)
|
|
251
|
+
and the drift. Callables are kept as they are, so they may close over app
|
|
252
|
+
state; like the model itself, a declaration is read on the main Ractor.
|
|
253
|
+
|
|
149
254
|
## License
|
|
150
255
|
|
|
151
256
|
MIT © Mikael Henriksson
|
|
@@ -26,6 +26,12 @@ defaults:
|
|
|
26
26
|
translate:
|
|
27
27
|
model: gpt-5-mini
|
|
28
28
|
batch_size: 20
|
|
29
|
+
# Optional cap on the characters of source text per batch (a batch closes at
|
|
30
|
+
# batch_size keys or max_batch_chars, whichever comes first). Useful for long
|
|
31
|
+
# prose; a single oversized value is still sent alone. null = keys only.
|
|
32
|
+
max_batch_chars: null
|
|
33
|
+
# Seconds to sleep after each batch (rate-limit courtesy). 0 disables it.
|
|
34
|
+
pause: 1.0
|
|
29
35
|
|
|
30
36
|
quality:
|
|
31
37
|
model: gpt-4o-mini
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/concern"
|
|
4
|
+
require "active_support/core_ext/class/attribute"
|
|
5
|
+
|
|
6
|
+
require_relative "../locallingo"
|
|
7
|
+
require_relative "content_leaves"
|
|
8
|
+
|
|
9
|
+
module Locallingo
|
|
10
|
+
# Opt-in ActiveRecord concern (`require "locallingo/active_record"`) for the
|
|
11
|
+
# common shape: a JSON(B) column keyed by locale plus a JSON(B) state column
|
|
12
|
+
# holding one Drift entry per target locale. Supports ActiveRecord >= 7.1.
|
|
13
|
+
#
|
|
14
|
+
# class Article < ApplicationRecord
|
|
15
|
+
# include Locallingo::Translatable
|
|
16
|
+
#
|
|
17
|
+
# translates_content :body, # { "en" => { … }, "de" => { … } }
|
|
18
|
+
# state: :translation_state, # { "de" => { "source_hash", "target_hash", "manual" } }
|
|
19
|
+
# source_locale: -> { "en" }, # String/Symbol, or a callable run on the record
|
|
20
|
+
# target_locales: -> { %w[de sv] },
|
|
21
|
+
# translatable: ->(path, value) { true } # optional filter on flattened String leaves
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# A `before_save` callback (#mark_hand_edits) flags target locales a person
|
|
25
|
+
# changed as manual, so Locallingo::RecordTranslator never overwrites them.
|
|
26
|
+
module Translatable
|
|
27
|
+
extend ActiveSupport::Concern
|
|
28
|
+
|
|
29
|
+
Definition = Data.define(:column, :state, :source_locale, :target_locales, :translatable, :drift)
|
|
30
|
+
|
|
31
|
+
included do
|
|
32
|
+
# The model's declaration: written once by translates_content in its class
|
|
33
|
+
# body at boot (main Ractor), and only read after that.
|
|
34
|
+
class_attribute :locallingo_translatable, instance_writer: false # rubocop:disable ThreadSafety/ClassAndModuleAttributes
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class_methods do
|
|
38
|
+
# The declaration is frozen as it is made, at boot: the column and state
|
|
39
|
+
# names and a String/Symbol/Array locale option (frozen copies; the
|
|
40
|
+
# caller's own objects are left alone), and the Drift. Callables (locale options,
|
|
41
|
+
# translatable:, hasher:) are kept as they are, so a declaration holding
|
|
42
|
+
# one is read on the main Ractor only, like the model itself.
|
|
43
|
+
def translates_content(column, state:, source_locale:, target_locales:, translatable: nil, hasher: nil)
|
|
44
|
+
self.locallingo_translatable = Definition.new(
|
|
45
|
+
column: Translatable.frozen_option(column.to_s), state: Translatable.frozen_option(state.to_s),
|
|
46
|
+
source_locale: Translatable.frozen_option(source_locale),
|
|
47
|
+
target_locales: Translatable.frozen_option(target_locales),
|
|
48
|
+
translatable: translatable || ->(_path, _value) { true },
|
|
49
|
+
drift: (hasher ? Drift.new(hasher:) : Drift.new).freeze
|
|
50
|
+
)
|
|
51
|
+
before_save :mark_hand_edits
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# A frozen copy of a non-callable declaration option (a String, or an Array
|
|
56
|
+
# of them); a callable or an already frozen value comes back as it is.
|
|
57
|
+
def self.frozen_option(value)
|
|
58
|
+
case value
|
|
59
|
+
when Array then value.map { |item| frozen_option(item) }.freeze
|
|
60
|
+
when String then value.frozen? ? value : value.dup.freeze
|
|
61
|
+
else value
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def translation_source_locale = resolve_translation_option(locallingo_translatable.source_locale).to_s
|
|
66
|
+
|
|
67
|
+
def translation_target_locales
|
|
68
|
+
Array(resolve_translation_option(locallingo_translatable.target_locales)).map(&:to_s)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def translation_source = translation_content[translation_source_locale]
|
|
72
|
+
def translation_target(locale) = translation_content[locale.to_s]
|
|
73
|
+
def translation_entry(locale) = translation_states[locale.to_s]
|
|
74
|
+
def translation_source_digest = translation_drift.digest(translation_source)
|
|
75
|
+
|
|
76
|
+
def translation_status(locale)
|
|
77
|
+
translation_drift.status(source: translation_source, target: translation_target(locale),
|
|
78
|
+
entry: translation_entry(locale))
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def translation_manual?(locale) = translation_drift.manual?(translation_entry(locale))
|
|
82
|
+
|
|
83
|
+
def translation_due?(locale, force: false, include_manual: false)
|
|
84
|
+
translation_drift.due?(source: translation_source, target: translation_target(locale),
|
|
85
|
+
entry: translation_entry(locale), force:, include_manual:)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Records the current source and target of +locale+ as its state entry
|
|
89
|
+
# (assigned, not saved).
|
|
90
|
+
def record_translation(locale, manual: false)
|
|
91
|
+
entry = translation_drift.record(source: translation_source, target: translation_target(locale), manual:)
|
|
92
|
+
self[locallingo_translatable.state] = translation_states.merge(locale.to_s => entry)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Flags every target locale a person changed as manual: a recorded target
|
|
96
|
+
# that no longer matches its target_hash, or an untracked locale written in
|
|
97
|
+
# this save. A cleared target is not flagged — it is missing and due again.
|
|
98
|
+
def mark_hand_edits
|
|
99
|
+
(translation_target_locales - [translation_source_locale]).each do |locale|
|
|
100
|
+
target = translation_target(locale)
|
|
101
|
+
next if translation_drift.blank?(target)
|
|
102
|
+
|
|
103
|
+
entry = translation_entry(locale)
|
|
104
|
+
next unless translation_drift.hand_edited?(target:, entry:) || (entry.nil? && translation_written?(locale))
|
|
105
|
+
|
|
106
|
+
# A recorded source_hash is kept: a hand edit does not make an
|
|
107
|
+
# outdated translation current.
|
|
108
|
+
manual = translation_drift.record(source: translation_source, target:, manual: true)
|
|
109
|
+
manual["source_hash"] = entry["source_hash"] if entry&.key?("source_hash")
|
|
110
|
+
self[locallingo_translatable.state] = translation_states.merge(locale => manual)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# The source's translatable String leaves, flattened (`steps[1].text`,
|
|
115
|
+
# `table[0][1]`). Unlike KeyFlattener, strings in nested arrays count.
|
|
116
|
+
def translation_segments
|
|
117
|
+
translation_leaves.filter_map do |path, (_, value)|
|
|
118
|
+
next unless value.is_a?(String) && !value.strip.empty?
|
|
119
|
+
|
|
120
|
+
[path, value] if locallingo_translatable.translatable.call(path, value)
|
|
121
|
+
end.to_h
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The target for translated +segments+ (`path => text`): a deep copy of the
|
|
125
|
+
# source with each translated leaf set in place, so every other value and
|
|
126
|
+
# container (numbers, nested lists, empty ones) keeps its place.
|
|
127
|
+
def build_translation_target(segments)
|
|
128
|
+
target = JSON.parse(JSON.generate(translation_source || {}))
|
|
129
|
+
leaves = translation_leaves
|
|
130
|
+
segments.each do |path, text|
|
|
131
|
+
*parents, last = leaves.fetch(path).first
|
|
132
|
+
(parents.empty? ? target : target.dig(*parents))[last] = text
|
|
133
|
+
end
|
|
134
|
+
target
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Column attributes that write +target+ for +locale+ plus its state entry.
|
|
138
|
+
def translation_attributes(locale, target, manual: false)
|
|
139
|
+
locale = locale.to_s
|
|
140
|
+
entry = translation_drift.record(source: translation_source, target:, manual:)
|
|
141
|
+
{
|
|
142
|
+
locallingo_translatable.column => translation_content.merge(locale => target),
|
|
143
|
+
locallingo_translatable.state => translation_states.merge(locale => entry)
|
|
144
|
+
}
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def translation_drift = locallingo_translatable.drift
|
|
148
|
+
|
|
149
|
+
private
|
|
150
|
+
|
|
151
|
+
def translation_leaves = ContentLeaves.call(translation_source || {}, label: "#{self.class.name}##{id}")
|
|
152
|
+
|
|
153
|
+
def translation_content = self[locallingo_translatable.column] || {}
|
|
154
|
+
def translation_states = self[locallingo_translatable.state] || {}
|
|
155
|
+
|
|
156
|
+
def translation_written?(locale)
|
|
157
|
+
column = locallingo_translatable.column
|
|
158
|
+
return false unless will_save_change_to_attribute?(column)
|
|
159
|
+
|
|
160
|
+
(attribute_in_database(column) || {})[locale] != translation_target(locale)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def resolve_translation_option(option)
|
|
164
|
+
option.respond_to?(:call) ? instance_exec(&option) : option
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
@@ -17,10 +17,16 @@ module Locallingo
|
|
|
17
17
|
BASE_SLEEP_DURATION = 1.0
|
|
18
18
|
|
|
19
19
|
# `log` is called as `log.call(message, level: :info)`.
|
|
20
|
-
|
|
20
|
+
#
|
|
21
|
+
# A batch closes at `batch_size` keys or `max_batch_chars` characters of
|
|
22
|
+
# source text, whichever comes first; a single oversized value goes alone.
|
|
23
|
+
# `pause` is the sleep after each batch (0 in a background job).
|
|
24
|
+
def initialize(provider:, model:, batch_size:, log:, max_batch_chars: nil, pause: BASE_SLEEP_DURATION)
|
|
21
25
|
@provider = provider
|
|
22
26
|
@model = model
|
|
23
27
|
@batch_size = batch_size
|
|
28
|
+
@max_batch_chars = max_batch_chars
|
|
29
|
+
@pause = pause
|
|
24
30
|
@log = log
|
|
25
31
|
end
|
|
26
32
|
|
|
@@ -66,7 +72,7 @@ module Locallingo
|
|
|
66
72
|
translations = {}
|
|
67
73
|
failed = []
|
|
68
74
|
|
|
69
|
-
keys.
|
|
75
|
+
batches(keys).each do |batch|
|
|
70
76
|
result = translate_batch(batch.to_h { |key| [key, @source[key]] })
|
|
71
77
|
batch.each do |key|
|
|
72
78
|
if result.key?(key) && !result[key].to_s.empty?
|
|
@@ -75,12 +81,29 @@ module Locallingo
|
|
|
75
81
|
failed << key
|
|
76
82
|
end
|
|
77
83
|
end
|
|
78
|
-
sleep(
|
|
84
|
+
sleep(@pause) if @pause.positive?
|
|
79
85
|
end
|
|
80
86
|
|
|
81
87
|
[translations, failed]
|
|
82
88
|
end
|
|
83
89
|
|
|
90
|
+
def batches(keys)
|
|
91
|
+
return keys.each_slice(@batch_size).to_a unless @max_batch_chars
|
|
92
|
+
|
|
93
|
+
keys.each_with_object([[]]) do |key, groups|
|
|
94
|
+
groups << [] if full?(groups.last, key)
|
|
95
|
+
groups.last << key
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def full?(batch, key)
|
|
100
|
+
return false if batch.empty?
|
|
101
|
+
|
|
102
|
+
batch.size >= @batch_size || chars(batch) + chars([key]) > @max_batch_chars
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def chars(batch) = batch.sum { |key| @source[key].to_s.length }
|
|
106
|
+
|
|
84
107
|
def translate_batch(payload)
|
|
85
108
|
return {} if payload.empty?
|
|
86
109
|
|
|
@@ -14,9 +14,12 @@ module Locallingo
|
|
|
14
14
|
#
|
|
15
15
|
# Typed readers expose everything the Manager/QualityChecker/CLI need, so no
|
|
16
16
|
# other class parses raw config hashes.
|
|
17
|
+
#
|
|
18
|
+
# The file is read when a Configuration is built (the CLI builds one per run)
|
|
19
|
+
# and never cached in the process, so an edit applies to the next run.
|
|
17
20
|
class Configuration
|
|
18
21
|
CONFIG_FILENAMES = %w[.locallingo.yml .locallingo.yaml].freeze
|
|
19
|
-
DEFAULT_CONFIG_PATH = File.expand_path("../../config/locallingo.default.yml", __dir__)
|
|
22
|
+
DEFAULT_CONFIG_PATH = File.expand_path("../../config/locallingo.default.yml", __dir__).freeze
|
|
20
23
|
|
|
21
24
|
attr_reader :root_path, :package, :data
|
|
22
25
|
|
|
@@ -26,10 +29,18 @@ module Locallingo
|
|
|
26
29
|
new(root_path:, package:)
|
|
27
30
|
end
|
|
28
31
|
|
|
29
|
-
|
|
32
|
+
# Resolve config from YAML +text+ (what a `.locallingo.yml` holds, ERB
|
|
33
|
+
# included) instead of the file under +root_path+, merged over the shipped
|
|
34
|
+
# defaults the same way.
|
|
35
|
+
def self.parse(text, root_path: Dir.pwd, package: nil)
|
|
36
|
+
new(root_path:, package:, source: text)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# +source+ is YAML text read in place of the config file (see .parse).
|
|
40
|
+
def initialize(root_path: Dir.pwd, package: nil, data: nil, source: nil)
|
|
30
41
|
@root_path = File.expand_path(root_path)
|
|
31
42
|
@package = package
|
|
32
|
-
@data = data || resolve
|
|
43
|
+
@data = data || resolve(source)
|
|
33
44
|
end
|
|
34
45
|
|
|
35
46
|
# --- locations -----------------------------------------------------------
|
|
@@ -54,6 +65,8 @@ module Locallingo
|
|
|
54
65
|
def provider = data.fetch("provider").to_sym
|
|
55
66
|
def translate_model = dig("translate", "model")
|
|
56
67
|
def batch_size = dig("translate", "batch_size")
|
|
68
|
+
def max_batch_chars = dig("translate", "max_batch_chars")
|
|
69
|
+
def pause = dig("translate", "pause") || 1.0
|
|
57
70
|
def quality_model = dig("quality", "model")
|
|
58
71
|
def quality_sample_size = dig("quality", "sample_size")
|
|
59
72
|
def british_spellings? = !!dig("quality", "british_spellings")
|
|
@@ -111,9 +124,10 @@ module Locallingo
|
|
|
111
124
|
def dig(*keys) = data.dig(*keys)
|
|
112
125
|
|
|
113
126
|
# Merge: shipped defaults <- user `defaults:` <- selected `packages:` entry.
|
|
114
|
-
def resolve
|
|
115
|
-
|
|
116
|
-
merged = deep_merge(
|
|
127
|
+
def resolve(source)
|
|
128
|
+
document, origin = user_config(source)
|
|
129
|
+
merged = deep_merge(shipped_defaults, document.fetch("defaults", {}) || {})
|
|
130
|
+
merged = deep_merge(merged, package_overrides(document, origin)) if package && origin
|
|
117
131
|
merged
|
|
118
132
|
end
|
|
119
133
|
|
|
@@ -121,20 +135,19 @@ module Locallingo
|
|
|
121
135
|
load_yaml(DEFAULT_CONFIG_PATH).fetch("defaults", {})
|
|
122
136
|
end
|
|
123
137
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
138
|
+
# The user's config and where it came from: the +source+ text, else the
|
|
139
|
+
# config file under root_path, else nothing.
|
|
140
|
+
def user_config(source)
|
|
141
|
+
return [parse_yaml(source), "the given YAML"] if source
|
|
127
142
|
|
|
128
|
-
load_yaml(file).fetch("defaults", {}) || {}
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
def package_overrides
|
|
132
143
|
file = config_file
|
|
133
|
-
|
|
144
|
+
file ? [load_yaml(file), file] : [{}, nil]
|
|
145
|
+
end
|
|
134
146
|
|
|
135
|
-
|
|
147
|
+
def package_overrides(document, origin)
|
|
148
|
+
packages = document.fetch("packages", []) || []
|
|
136
149
|
entry = packages.find { |p| p["path"] == package }
|
|
137
|
-
raise Error, "No package #{package.inspect} in #{
|
|
150
|
+
raise Error, "No package #{package.inspect} in #{origin}" unless entry
|
|
138
151
|
|
|
139
152
|
entry.except("path")
|
|
140
153
|
end
|
|
@@ -145,10 +158,19 @@ module Locallingo
|
|
|
145
158
|
.find { |path| File.exist?(path) }
|
|
146
159
|
end
|
|
147
160
|
|
|
148
|
-
def load_yaml(path)
|
|
149
|
-
|
|
161
|
+
def load_yaml(path) = parse_yaml(File.read(path))
|
|
162
|
+
|
|
163
|
+
def parse_yaml(text)
|
|
164
|
+
YAML.safe_load(ERB.new(text).result(erb_binding), aliases: true) || {}
|
|
150
165
|
end
|
|
151
166
|
|
|
167
|
+
# ERB's default binding is TOPLEVEL_BINDING, which a non-main Ractor cannot
|
|
168
|
+
# reach; a blank object's binding evaluates the same templates. The string
|
|
169
|
+
# form, not a block: a block would resolve constants from inside
|
|
170
|
+
# Locallingo::Configuration (`Settings` would be Locallingo::Settings), the
|
|
171
|
+
# string resolves them from the top level, as TOPLEVEL_BINDING did.
|
|
172
|
+
def erb_binding = Object.new.instance_eval("binding", __FILE__, __LINE__)
|
|
173
|
+
|
|
152
174
|
def deep_merge(base, override)
|
|
153
175
|
base.merge(override) do |_key, base_val, override_val|
|
|
154
176
|
if base_val.is_a?(Hash) && override_val.is_a?(Hash)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Locallingo
|
|
4
|
+
# Walks a nested Hash/Array value to its leaves, keyed by a KeyFlattener-style
|
|
5
|
+
# path (`steps[1].text`, `table[0][1]`). Unlike KeyFlattener, strings inside
|
|
6
|
+
# nested arrays count, and each leaf carries the Hash keys and Array indices
|
|
7
|
+
# leading to it, so a value can be set back in place without parsing paths.
|
|
8
|
+
module ContentLeaves
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# `path => [parts, value]`. Two leaves with the same path (a literal "a.b"
|
|
12
|
+
# key next to a → b) raise: one would be silently dropped from translation.
|
|
13
|
+
def call(value, label: "content")
|
|
14
|
+
collect(value, [], {}, label)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def collect(value, parts, leaves, label)
|
|
18
|
+
case value
|
|
19
|
+
when Hash then value.each { |key, item| collect(item, parts + [key.to_s], leaves, label) }
|
|
20
|
+
when Array then value.each_with_index { |item, index| collect(item, parts + [index], leaves, label) }
|
|
21
|
+
else
|
|
22
|
+
path = path(parts)
|
|
23
|
+
raise Error, "Ambiguous translation path #{path.inspect} in #{label}" if leaves.key?(path)
|
|
24
|
+
|
|
25
|
+
leaves[path] = [parts, value]
|
|
26
|
+
end
|
|
27
|
+
leaves
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def path(parts)
|
|
31
|
+
parts.each_with_index.map do |part, index|
|
|
32
|
+
next "[#{part}]" if part.is_a?(Integer)
|
|
33
|
+
|
|
34
|
+
index.zero? ? part : ".#{part}"
|
|
35
|
+
end.join
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private_class_method :collect, :path
|
|
39
|
+
end
|
|
40
|
+
end
|