translation_diff 1.0.0

data/Gemfile

@@ -0,0 +1,61 @@
+source "https://rubygems.org"
+
+gemspec
+
+# Not runtime dependencies of the gem (see the gemspec) -- Configuration#
+# redis_pool requires them lazily, and Stores::Redis/RateLimiters::Redis
+# duck-type against whatever a caller's connection pool yields. They are
+# only here so the test suite, which builds real pools against these
+# classes, has them available.
+gem "connection_pool", "~> 2.4", require: false
+gem "redis", "~> 5.0", require: false
+gem "redis-namespace", "~> 1.11", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- RateLimiters::Redis
+# requires it lazily on the first check, so an application that configures no
+# rate limit never needs it installed. It is only here so the test suite,
+# which exercises the limiter against the real Ratelimit class rather than a
+# stand-in, has it available.
+gem "ratelimit", "~> 1.1", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- lib/ only ever
+# needs CGI.escape, which cgi/escape (in Ruby's default load path) still
+# provides. This is here because the Google provider's test decodes the
+# query string it sent, and Ruby 4.0 removed CGI.parse from the default
+# load path; "install cgi gem" is Ruby's own suggested fix.
+gem "cgi", "~> 0.5", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- the Amazon
+# provider requires it lazily when it signs its first request, so an
+# application using another provider never needs it installed. It is here so
+# the test suite, which signs against the real library rather than a
+# stand-in, has it available.
+gem "aws-sigv4", "~> 1.12", require: false
+
+# Not runtime dependencies of the gem (see the gemspec) -- TranslationDiff::Stores::ActiveRecord
+# and TranslationDiff::RateLimiters::ActiveRecord require active_record lazily on first use, so an
+# application caching in Redis never needs it installed. They are here so the
+# suite can exercise the stores against a real database rather than a stand-in.
+gem "activerecord", "~> 8.1", require: false
+gem "sqlite3", "~> 2.9", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- only the CI job that
+# sets TRANSLATION_DIFF_DATABASE_URL ever opens a Postgres connection, where
+# the concurrency the SQL store and rate limiter rest on can actually be
+# tested. SQLite has one writer, so most of the suite never needs this gem.
+gem "pg", "~> 1.5", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- only the CI job that
+# sets TRANSLATION_DIFF_DATABASE_URL to a MySQL database ever opens one, where
+# the missing supports_insert_conflict_target? behaviour actually bites. Trilogy
+# over mysql2: it is a pure Ruby/C socket client with no libmysqlclient headers
+# to install, so it builds on a bare CI runner and on this machine alike.
+gem "trilogy", "~> 2.9", require: false
+
+# Not a runtime dependency of the gem (see the gemspec) -- the generator under
+# lib/generators/ is loaded only when Rails loads generators, so a non-Rails
+# application never needs it installed. It is here so
+# test/translation_diff/install_generator_test.rb can load it and check the
+# generated migration against the schema the rest of the suite runs against,
+# instead of skipping itself for want of Rails::Generators::Base.
+gem "railties", "~> 8.1", require: false

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Islam Gagiev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,158 @@
+# TranslationDiff
+
+A translation cache that helps translate only changes between revisions of long texts.
+
+## Why TranslationDiff?
+
+Assume your project contains a significant amount of products descriptions which:
+- Require retranslation each time user edits them.
+- Have a lot of equal parts (like return policy).
+- Change frequently.
+
+If your user changes a single word within the long description, you will be charged for the retranslation of the whole text.
+
+Much better approach is to try to translate every repeated structural element (sentence) in your texts array just once to save money. This gem helps to make it done.
+
+## Show me the code
+
+```ruby
+# The simplest translation
+TranslationDiff.configure do |config|
+  config.deepl_api_key = ENV["DEEPL_API_KEY"]
+  config.redis_url = ENV["REDIS_URL"]
+end
+
+TranslationDiff.translate("Привет.", from: "ru", to: "en")
+```
+
+```ruby
+# Nested structures come back the same shape
+TranslationDiff.translate("test", from: "en", to: "es")
+TranslationDiff.translate(%w[test language], from: "en", to: "es")
+TranslationDiff.translate(
+  { title: "test", values: { type: "frequent" } }, from: "en", to: "es"
+)
+```
+
+```ruby
+# HTML markup is preserved
+TranslationDiff.translate("<b>Black</b>", from: "en", to: "es")
+```
+
+```ruby
+# class="notranslate" marks a span to protect (provider support varies -- see the caveats below)
+TranslationDiff.translate(
+  '<span class="notranslate">Bold Mountain</span> is a good place.', from: "en", to: "ru"
+)
+```
+
+```ruby
+# Switch provider for one call, without touching the global configuration
+TranslationDiff.translate(blog_post, from: "en", to: "de", provider: :google)
+```
+
+```ruby
+# Any keyword other than from:, to:, provider:, config: and assume_supported: is forwarded to
+# the provider. assume_supported: is this library's own decision, not a vendor's, so it's
+# reserved rather than forwarded -- it must never reach a payload.
+TranslationDiff.translate(contract, from: "en", to: "de", formality: :more)
+```
+
+```ruby
+# An isolated context for its own configuration -- global config.deepl_api_key stays untouched
+formal = TranslationDiff.context do |config|
+  config.provider = :deepl
+  config.deepl_api_key = ENV["DEEPL_AUTH_KEY"]
+end
+formal.translate(contract, from: "en", to: "de", formality: :more)
+```
+
+```ruby
+# See what a call would send and find cached, without calling the provider or writing anything
+preview = TranslationDiff.preview(contract, from: "en", to: "de")
+preview.sendable_sentences # => sentences not yet cached
+preview.cached_sentences   # => sentences already cached
+```
+
+```ruby
+# Redis-backed cache, shared across processes
+TranslationDiff.configure { |config| config.redis_url = ENV["REDIS_URL"] }
+```
+
+```ruby
+# A rate limit, in characters per interval
+TranslationDiff.configure { |config| config.rate_limit = 100_000 }
+```
+
+```ruby
+# Instrumentation, through anything satisfying the ActiveSupport::Notifications interface
+TranslationDiff.configure { |config| config.instrumenter = ActiveSupport::Notifications }
+```
+
+## Providers
+
+Every provider declares what it can do through
+`TranslationDiff::Capabilities` -- there is nowhere else these numbers live,
+so this table is generated from the same source the library reads at
+runtime:
+
+| Provider | `config.provider` | Auth option(s) | Batch size | Request size (escaped chars) | HTML support | `notranslate` | Detects language | Reports billing |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| Null | `:null` | none | 1,000,000 | 1,000,000 | no | no | no | no |
+| DeepL | `:deepl` (default) | `deepl_api_key` | 50 | 1,700 | yes (`tag_handling`) | yes | yes | yes |
+| Google | `:google` | `google_api_key` | 128 | 5,000 | yes (`format`) | yes | yes | no |
+| Azure | `:azure` | `azure_api_key` | 1,000 | 50,000 | yes (`textType`) | yes | yes | yes |
+| ModernMT | `:modernmt` | `modernmt_api_key` | 128 | 5,000 | yes (`format`) | no | yes | yes |
+| LibreTranslate | `:libretranslate` | `libretranslate_api_base` | 50 | 5,000 | yes (`format`) | no | yes | no |
+| Amazon | `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` | 1 | 10,000 | no | no | yes | no |
+
+**Amazon:** translates one text per call (no batch form of `TranslateText`) and has no HTML mode, so `notranslate` is not honoured.
+
+**LibreTranslate:** preserves markup but does not honour `notranslate` either -- measured against a real instance, not assumed.
+
+See [Providers](docs/providers.md) for configuring each one, the full capabilities explanation, and writing your own.
+
+## Features
+
+- **Content-hash caching:** a small edit to a long document is billed for the edit, not the whole document
+- **Six built-in providers** -- DeepL, Google Cloud Translation, Azure AI Translator, ModernMT, LibreTranslate, Amazon Translate -- or bring your own by subclassing a small base class
+- **HTML aware:** markup is preserved, and `class="notranslate"` can protect a span (provider support varies -- see the caveats below)
+- **Any shape:** strings, arrays, and deep hashes go in and come back translated in the same shape
+- **Three cache stores:** `Stores::Memory` out of the box, `Stores::Redis` once you configure `redis_url`, `Stores::ActiveRecord` to cache in your own database instead -- see [SQL cache](docs/sql-cache.md)
+- **Isolated contexts:** `TranslationDiff.context` for multi-tenant apps and per-request provider overrides, without touching the global configuration
+- **Pluggable sentence segmenter:** `pragmatic_segmenter` by default, with a zero-dependency `Simple` alternative
+- **HTTP retries, timeouts, and backoff** on every REST-backed provider, via `faraday` and `faraday-retry`
+- **One error hierarchy** under `TranslationDiff::Error`, carrying the provider name and HTTP status
+- **Optional rate limiting and instrumentation** -- credentials and translated content never appear in a log line this gem writes (the SQL cache store is the one exception worth knowing before you adopt it -- see [SQL cache](docs/sql-cache.md#what-ends-up-in-your-log))
+
+## Installation
+
+Ruby 3.4 or newer is required.
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'translation_diff'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install translation_diff
+
+This gem loads `ox`, `pragmatic_segmenter`, `faraday`, and `faraday-retry` at require time. `aws-sigv4`, `activerecord`, `redis`, `connection_pool`, `redis-namespace`, and `ratelimit` are yours to add, only if you use the feature that needs them -- see [Dependencies](docs/configuration.md#dependencies).
+
+## Documentation
+
+[Configuration](docs/configuration.md) · [Providers](docs/providers.md) · [Languages](docs/languages.md) · [Caching](docs/caching.md) · [SQL cache](docs/sql-cache.md) · [Contracts](docs/contracts.md) · [Instrumentation](docs/instrumentation.md) · [Errors](docs/errors.md) · [How it works](docs/how-it-works.md) · [Upgrading & development](docs/development.md)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Halvanhelv/translation_diff.
+
+## License
+
+Released under the [MIT License](LICENSE.txt).

data/Rakefile

@@ -0,0 +1,41 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+  # The dependencies are noisy under -w; their warnings drown out our own.
+  t.warning = false
+end
+
+task default: :test
+
+namespace :languages do
+  desc "Re-fetch every provider's language lists from its vendor"
+  task :refresh do
+    require "translation_diff"
+
+    not_shipped = TranslationDiff::Languages::NOT_SHIPPED
+    unless not_shipped.empty?
+      puts "not shipping data for #{not_shipped.join(', ')}: a vendor credential or a private instance " \
+           "would make the file unshareable"
+    end
+
+    skip = [:null, *not_shipped]
+    providers = TranslationDiff::Providers.names.reject { |name| skip.include?(name) }.map do |name|
+      TranslationDiff::Providers.build(name, TranslationDiff.config)
+    rescue TranslationDiff::ConfigurationError => e
+      warn "skipping #{name}: #{e.message}"
+      nil
+    end.compact
+
+    report = TranslationDiff::Languages::Refresh.call(providers: providers)
+    puts "updated: #{report[:updated].join(', ')}" unless report[:updated].empty?
+    puts "skipped: #{report[:skipped].join(', ')}" unless report[:skipped].empty?
+    report[:failed].each { |name, message| warn "failed: #{name}: #{message}" }
+  end
+end
+
+# The task itself ships in lib/, so a host application's own `rake` can load it too -- this just reuses it here.
+load File.expand_path("lib/translation_diff/tasks/translation_diff.rake", __dir__)

data/data/languages/azure.json

@@ -0,0 +1,285 @@
+{
+  "provider": "azure",
+  "captured_at": "2026-09-10",
+  "endpoint": "https://api.cognitive.microsofttranslator.com/languages?api-version=3.0&scope=translation",
+  "source": [
+    "af",
+    "am",
+    "ar",
+    "as",
+    "az",
+    "ba",
+    "be",
+    "bg",
+    "bho",
+    "bn",
+    "bo",
+    "brx",
+    "bs",
+    "ca",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "doi",
+    "dsb",
+    "dv",
+    "el",
+    "en",
+    "es",
+    "es-mx",
+    "et",
+    "eu",
+    "fa",
+    "fi",
+    "fil",
+    "fj",
+    "fo",
+    "fr",
+    "fr-ca",
+    "ga",
+    "gl",
+    "gom",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hne",
+    "hr",
+    "hsb",
+    "ht",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "ikt",
+    "is",
+    "it",
+    "iu",
+    "iu-latn",
+    "ja",
+    "ka",
+    "kk",
+    "km",
+    "kmr",
+    "kn",
+    "ko",
+    "ks",
+    "ku",
+    "ky",
+    "lb",
+    "ln",
+    "lo",
+    "lt",
+    "lug",
+    "lv",
+    "lzh",
+    "mai",
+    "mg",
+    "mi",
+    "mk",
+    "ml",
+    "mn-cyrl",
+    "mn-mong",
+    "mni",
+    "mr",
+    "ms",
+    "mt",
+    "mww",
+    "my",
+    "nb",
+    "ne",
+    "nl",
+    "nso",
+    "nya",
+    "or",
+    "otq",
+    "pa",
+    "pl",
+    "prs",
+    "ps",
+    "pt",
+    "pt-pt",
+    "ro",
+    "ru",
+    "run",
+    "rw",
+    "sd",
+    "si",
+    "sk",
+    "sl",
+    "sm",
+    "sn",
+    "so",
+    "sq",
+    "sr-cyrl",
+    "sr-latn",
+    "st",
+    "sv",
+    "sw",
+    "ta",
+    "te",
+    "th",
+    "ti",
+    "tk",
+    "tlh-latn",
+    "tlh-piqd",
+    "tn",
+    "to",
+    "tr",
+    "tt",
+    "ty",
+    "ug",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "xh",
+    "yo",
+    "yua",
+    "yue",
+    "zh-hans",
+    "zh-hant",
+    "zu"
+  ],
+  "target": [
+    "af",
+    "am",
+    "ar",
+    "as",
+    "az",
+    "ba",
+    "be",
+    "bg",
+    "bho",
+    "bn",
+    "bo",
+    "brx",
+    "bs",
+    "ca",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "doi",
+    "dsb",
+    "dv",
+    "el",
+    "en",
+    "es",
+    "es-mx",
+    "et",
+    "eu",
+    "fa",
+    "fi",
+    "fil",
+    "fj",
+    "fo",
+    "fr",
+    "fr-ca",
+    "ga",
+    "gl",
+    "gom",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hne",
+    "hr",
+    "hsb",
+    "ht",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "ikt",
+    "is",
+    "it",
+    "iu",
+    "iu-latn",
+    "ja",
+    "ka",
+    "kk",
+    "km",
+    "kmr",
+    "kn",
+    "ko",
+    "ks",
+    "ku",
+    "ky",
+    "lb",
+    "ln",
+    "lo",
+    "lt",
+    "lug",
+    "lv",
+    "lzh",
+    "mai",
+    "mg",
+    "mi",
+    "mk",
+    "ml",
+    "mn-cyrl",
+    "mn-mong",
+    "mni",
+    "mr",
+    "ms",
+    "mt",
+    "mww",
+    "my",
+    "nb",
+    "ne",
+    "nl",
+    "nso",
+    "nya",
+    "or",
+    "otq",
+    "pa",
+    "pl",
+    "prs",
+    "ps",
+    "pt",
+    "pt-pt",
+    "ro",
+    "ru",
+    "run",
+    "rw",
+    "sd",
+    "si",
+    "sk",
+    "sl",
+    "sm",
+    "sn",
+    "so",
+    "sq",
+    "sr-cyrl",
+    "sr-latn",
+    "st",
+    "sv",
+    "sw",
+    "ta",
+    "te",
+    "th",
+    "ti",
+    "tk",
+    "tlh-latn",
+    "tlh-piqd",
+    "tn",
+    "to",
+    "tr",
+    "tt",
+    "ty",
+    "ug",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "xh",
+    "yo",
+    "yua",
+    "yue",
+    "zh-hans",
+    "zh-hant",
+    "zu"
+  ]
+}