translation_diff 1.0.0

data/data/languages/modernmt.json

@@ -0,0 +1,413 @@
+{
+  "provider": "modernmt",
+  "captured_at": "2026-09-10",
+  "endpoint": "https://api.modernmt.com/translate/languages",
+  "source": [
+    "ace",
+    "af",
+    "ak",
+    "als",
+    "am",
+    "ar",
+    "as",
+    "ast",
+    "awa",
+    "ayr",
+    "az",
+    "azb",
+    "azj",
+    "ba",
+    "ban",
+    "be",
+    "bem",
+    "bg",
+    "bho",
+    "bjn",
+    "bm",
+    "bn",
+    "bo",
+    "bs",
+    "bug",
+    "ca",
+    "ceb",
+    "cjk",
+    "ckb",
+    "crh",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dik",
+    "diq",
+    "dyu",
+    "dz",
+    "ee",
+    "el",
+    "en",
+    "eo",
+    "es",
+    "es-419",
+    "es-es",
+    "et",
+    "fi",
+    "fj",
+    "fo",
+    "fon",
+    "fr",
+    "fur",
+    "fuv",
+    "ga",
+    "gaz",
+    "gd",
+    "gl",
+    "gn",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hne",
+    "hr",
+    "ht",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "ilo",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kab",
+    "kac",
+    "kam",
+    "kas",
+    "kbp",
+    "kea",
+    "kg",
+    "khk",
+    "ki",
+    "kk",
+    "km",
+    "kmb",
+    "kmr",
+    "kn",
+    "knc",
+    "ko",
+    "ks",
+    "ky",
+    "la",
+    "lb",
+    "lg",
+    "li",
+    "lij",
+    "lmo",
+    "ln",
+    "lo",
+    "lt",
+    "ltg",
+    "lua",
+    "luo",
+    "lus",
+    "lv",
+    "lvs",
+    "mag",
+    "mai",
+    "mg",
+    "mi",
+    "min",
+    "mk",
+    "ml",
+    "mn",
+    "mni",
+    "mos",
+    "mr",
+    "ms",
+    "mt",
+    "my",
+    "nb",
+    "ne",
+    "nl",
+    "nn",
+    "nso",
+    "nus",
+    "ny",
+    "oc",
+    "or",
+    "pa",
+    "pag",
+    "pap",
+    "pbt",
+    "pes",
+    "pl",
+    "plt",
+    "prs",
+    "ps",
+    "pt",
+    "pt-br",
+    "pt-pt",
+    "quy",
+    "rn",
+    "ro",
+    "ru",
+    "rw",
+    "sa",
+    "sat",
+    "sc",
+    "scn",
+    "sd",
+    "sg",
+    "shn",
+    "si",
+    "sk",
+    "sl",
+    "sm",
+    "sn",
+    "so",
+    "sq",
+    "sr",
+    "ss",
+    "st",
+    "su",
+    "sv",
+    "sw",
+    "szl",
+    "ta",
+    "taq",
+    "te",
+    "tg",
+    "th",
+    "ti",
+    "tk",
+    "tl",
+    "tn",
+    "tpi",
+    "tr",
+    "ts",
+    "tt",
+    "tum",
+    "tw",
+    "tzm",
+    "ug",
+    "uk",
+    "umb",
+    "ur",
+    "uzn",
+    "vec",
+    "vi",
+    "war",
+    "wo",
+    "xh",
+    "ydd",
+    "yo",
+    "zh",
+    "zh-cn",
+    "zh-tw",
+    "zsm",
+    "zu"
+  ],
+  "target": [
+    "ace",
+    "af",
+    "ak",
+    "als",
+    "am",
+    "ar",
+    "as",
+    "ast",
+    "awa",
+    "ayr",
+    "az",
+    "azb",
+    "azj",
+    "ba",
+    "ban",
+    "be",
+    "bem",
+    "bg",
+    "bho",
+    "bjn",
+    "bm",
+    "bn",
+    "bo",
+    "bs",
+    "bug",
+    "ca",
+    "ceb",
+    "cjk",
+    "ckb",
+    "crh",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dik",
+    "diq",
+    "dyu",
+    "dz",
+    "ee",
+    "el",
+    "en",
+    "eo",
+    "es",
+    "es-419",
+    "es-es",
+    "et",
+    "fi",
+    "fj",
+    "fo",
+    "fon",
+    "fr",
+    "fur",
+    "fuv",
+    "ga",
+    "gaz",
+    "gd",
+    "gl",
+    "gn",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hne",
+    "hr",
+    "ht",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "ilo",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kab",
+    "kac",
+    "kam",
+    "kas",
+    "kbp",
+    "kea",
+    "kg",
+    "khk",
+    "ki",
+    "kk",
+    "km",
+    "kmb",
+    "kmr",
+    "kn",
+    "knc",
+    "ko",
+    "ks",
+    "ky",
+    "la",
+    "lb",
+    "lg",
+    "li",
+    "lij",
+    "lmo",
+    "ln",
+    "lo",
+    "lt",
+    "ltg",
+    "lua",
+    "luo",
+    "lus",
+    "lv",
+    "lvs",
+    "mag",
+    "mai",
+    "mg",
+    "mi",
+    "min",
+    "mk",
+    "ml",
+    "mn",
+    "mni",
+    "mos",
+    "mr",
+    "ms",
+    "mt",
+    "my",
+    "nb",
+    "ne",
+    "nl",
+    "nn",
+    "nso",
+    "nus",
+    "ny",
+    "oc",
+    "or",
+    "pa",
+    "pag",
+    "pap",
+    "pbt",
+    "pes",
+    "pl",
+    "plt",
+    "prs",
+    "ps",
+    "pt",
+    "pt-br",
+    "pt-pt",
+    "quy",
+    "rn",
+    "ro",
+    "ru",
+    "rw",
+    "sa",
+    "sat",
+    "sc",
+    "scn",
+    "sd",
+    "sg",
+    "shn",
+    "si",
+    "sk",
+    "sl",
+    "sm",
+    "sn",
+    "so",
+    "sq",
+    "sr",
+    "ss",
+    "st",
+    "su",
+    "sv",
+    "sw",
+    "szl",
+    "ta",
+    "taq",
+    "te",
+    "tg",
+    "th",
+    "ti",
+    "tk",
+    "tl",
+    "tn",
+    "tpi",
+    "tr",
+    "ts",
+    "tt",
+    "tum",
+    "tw",
+    "tzm",
+    "ug",
+    "uk",
+    "umb",
+    "ur",
+    "uzn",
+    "vec",
+    "vi",
+    "war",
+    "wo",
+    "xh",
+    "ydd",
+    "yo",
+    "zh",
+    "zh-cn",
+    "zh-tw",
+    "zsm",
+    "zu"
+  ]
+}

data/docs/caching.md

@@ -0,0 +1,185 @@
+# Caching
+
+## What a cache key is made of
+
+One entry per sentence, keyed by the provider's `cache_key`, the lowercased
+source and target language codes, a digest of the provider options that call
+passed (`formality:`, a glossary id, ...), and a digest of the sentence
+itself. `Stores::Redis` prefixes all of that with `cache_namespace`.
+
+**No provider's `*_api_base` option is part of the key.** Two configurations
+pointing `deepl_api_base` (or any other provider's `_api_base`) at different
+endpoints share cache entries. For DeepL's own free and paid hosts that is
+correct -- they return the same translations -- but a self-hosted or proxied
+endpoint may not, and it would be served, and would serve, the real
+service's entries. Give such a configuration its own `cache_namespace` (or
+its own Redis database). The key format is left alone here on purpose:
+changing its shape invalidates every entry already cached, everywhere, at
+once.
+
+**A cache entry written before a bug fix keeps serving what the bug
+produced.** The key above is built from the source sentence, never from the
+value stored under it, so fixing what a provider's reply decodes to does not
+invalidate what is already cached -- an entry written under the HTML-entity
+double-escaping fixed in the Unreleased CHANGELOG entry is served exactly as
+it was written until it expires. Give the configuration a new
+`cache_namespace`, or let `cache_ttl` lapse, to force every sentence to be
+retranslated under the fix.
+
+Both read and write the same cache, keyed per provider, so switching one
+never serves you the other's translations.
+
+`TranslationDiff::SentenceCache` is the class that builds the key and does
+both the read and the write.
+
+## The options digest is lossy, on purpose
+
+The per-call options are canonicalised to a string before they are digested,
+and that canonical form flattens more than it distinguishes. Nesting is not
+recorded, so `["x", ["y", "z"]]` and `["x", "y", "z"]` canonicalise
+identically; neither is emptiness typed, so `tags: []` and `tags: {}` do
+too. Two calls whose options differ only in one of those ways share a cache
+entry.
+
+This is a known property, not an oversight. The pipeline this replaced
+collides on exactly the same inputs -- that was checked, not assumed -- so
+reproducing it was the choice that left every warm cache warm. Fixing it
+would give those calls new keys and re-translate everything already cached
+under the old ones, for a distinction no provider option this gem ships
+actually makes. If you pass an option where that distinction matters, give
+the configuration its own `cache_namespace`.
+
+A value the canonical form cannot render at all -- anything that is not a
+String, Symbol, Numeric, `true`, `false`, `nil`, or an Array or Hash of
+those -- raises `TranslationDiff::SentenceCache::Error` rather than being
+guessed at. A key that is silently wrong costs you the whole cache and tells
+you nothing.
+
+No options at all contributes no field to the key, which is the four-field
+key every already-warm cache is keyed on.
+
+## Asking what a call would do, without doing it
+
+`TranslationDiff.preview` answers what a `translate` call would send and
+find cached, without calling a provider and without writing anything: how
+many sentences it would send, how many the cache already has, and how many
+characters that is. It reads the same store, through the same
+`SentenceCache`, keyed the same way -- see [What a cache key is made
+of](#what-a-cache-key-is-made-of) above -- so a preview and the call it
+predicts always agree.
+
+```ruby
+preview = TranslationDiff.preview(article_body, from: "en", to: "es")
+preview.sendable_sentences   # => 1, not yet cached
+preview.cached_sentences     # => 4, already cached
+preview.sendable_characters  # => 23
+preview.characters           # => 412, the total this call would consider
+```
+
+`sendable_sentences` and `cached_sentences` are the same two counts the
+`cache` event reports as `misses` and `hits`; `characters` is the same total
+the `translate` event reports. A preview and the call it predicts are
+answering the same question through the same numbers, so "this edit will
+send 23 of 412 characters" and what the events for that call later report
+should agree.
+
+**`from:` is required wherever there is anything to preview.** `translate`
+can leave `from:` unset and pay for one `#detect` request to find it; a
+preview never calls the provider, so it cannot pay for that request either.
+Passing `to:` alone raises `TranslationDiff::Previewer::Error`, naming the
+provider and telling you to pass `from:` explicitly -- unless the document
+holds nothing translatable, or the source and target already match, in
+which case there is nothing to preview and an empty result comes back
+regardless of `from:`.
+
+This is the supported way to ask an editor's question before it becomes a
+bill -- show "this edit will send 1 sentence" before the author saves:
+
+```ruby
+preview = TranslationDiff.preview(edited_body, from: "en", to: "es")
+"This edit will send #{preview.sendable_sentences} sentence#{'s' unless preview.sendable_sentences == 1}."
+```
+
+## The cache store contract
+
+`config.cache` accepts either a registered name (`:redis`, `:memory`,
+`:active_record`) or an object satisfying this contract directly:
+
+```ruby
+# Reads several keys at once, returning an array the same length as keys,
+# with nil in a missing key's position.
+def read_multi(keys); end
+
+# Writes one key. The second write of the same key replaces the first.
+def write(key, value); end
+
+# Writes several pairs at once. Optional -- see "write_multi is optional" below.
+def write_multi(pairs); end
+```
+
+`test/support/cache_store_contract.rb` is the executable form of this
+contract: include `CacheStoreContract` in a test class that defines
+`#store`. It only exercises `read_multi` and `write` -- the two required
+methods -- so a store that implements only those two still passes it.
+`test/support/batching_cache_store_contract.rb` holds the optional half:
+include `BatchingCacheStoreContract` too, alongside `CacheStoreContract`,
+once `#store` also implements `write_multi`.
+
+Three stores ship with this gem: `TranslationDiff::Stores::Memory`, the
+default -- a bounded, in-process LRU, not thread-safe by design, evicting by
+`cache_max_size` rather than by time; `TranslationDiff::Stores::Redis`,
+built from `redis_url` when that is set, expiring entries after `cache_ttl`
+and namespacing every key under `cache_namespace`; and
+`TranslationDiff::Stores::ActiveRecord`, opt-in, caching in the
+application's own database -- see [SQL cache](sql-cache.md). Neither `redis`
+nor `connection_pool` nor `redis-namespace` is a dependency of this gem --
+`Stores::Redis` takes anything answering to `#with` the way
+`ConnectionPool` does, and yields anything `Redis::Namespace` accepts.
+
+## `write_multi` is optional
+
+A store need not implement `write_multi`. `SentenceCache#store` checks: a
+store that answers to it gets one call carrying every translated sentence
+from the batch; a store that does not is called once per sentence through
+`write` instead, exactly as it always was. A custom cache store written
+against the contract before `write_multi` existed keeps working unchanged
+-- that is what "optional" means here.
+
+All three shipped stores implement it: `Stores::Memory` loops over the
+pairs (there is no round trip to save in-process); `Stores::Redis`
+pipelines the writes; `Stores::ActiveRecord` upserts the whole batch in
+one statement.
+
+### The three write paths fail differently
+
+Nobody had written this down before: what a partial failure leaves cached
+depends on which of these shapes wrote it.
+
+- **No `write_multi` (the per-key path), and `Stores::Memory`'s loop.**
+  Sentences are written one at a time, in order. A failure at sentence N
+  leaves 1..N-1 written, N failed, and N+1.. never attempted.
+- **`Stores::Redis#write_multi`.** A Redis pipeline is not a
+  transaction: each `SETEX` in it runs independently of the others, so a
+  failure in one does not stop its siblings from landing. Which of the
+  batch actually landed does not follow the sentence order the way the
+  per-key path's does.
+- **`Stores::ActiveRecord#write_multi`.** One `upsert_all` statement for
+  the whole batch. It either lands as a whole or it does not -- there is no
+  partial batch to reason about.
+
+A caller that needs to know which sentences got cached after a failure
+needs to know which of these three shapes wrote them; the answer is not the
+same for all three.
+
+None of the three ever reaches the caller as an exception, though. The
+cache is an optimisation on top of a translation that has already been
+paid for at the provider: `Translator#fill` rescues whatever error surfaces
+here, logs it, fires a `cache_error` event (provider and error class only,
+never the text -- see [Instrumentation](instrumentation.md)), and returns
+the translation regardless. This holds for all three shapes and every
+store, not only `Stores::ActiveRecord` -- a `Stores::Memory` bug, a
+dropped Redis connection, a SQL write blocked by a read-only replica (see
+[Rails replica routing](sql-cache.md#rails-replica-routing)) all behave the
+same way from the caller's side. What differs between the three shapes
+above is only what ends up cached, never whether the translation comes
+back.