translation_diff 1.0.0

data/lib/translation_diff/translator.rb

@@ -0,0 +1,103 @@
+# One translation, coordinated: each step hands its value to the next, and a translation rides home on its own segment.
+class TranslationDiff::Translator
+  # Its own class, so rescuing a mis-driven translator cannot also swallow a provider or a cache failure.
+  class Error < TranslationDiff::Error; end
+
+  include TranslationDiff::Instrumentation
+  include TranslationDiff::CallPreparation
+
+  attr_reader :config
+
+  # `provider:`, `config:` and `assume_supported:` are reserved; every other keyword reaches the provider untouched.
+  def initialize(values, from: nil, to: nil, provider: nil, config: nil, assume_supported: false, **options)
+    raise ArgumentError, "a translation needs a target language: pass `to:` a language code." if to.nil?
+
+    @values = values
+    @from = from
+    @to = to
+    @options = options
+    @config = config || TranslationDiff.config
+    @requested_provider = provider
+    @assume_supported = assume_supported
+  end
+
+  # Hands back the caller's value untouched unless something in it was actually translated.
+  def call
+    document = TranslationDiff::Document.new(TranslationDiff::Leaves.collapse_nils(@values))
+    passages = document.strings.map { |string| passage(string) }
+    segments = passages.flat_map(&:segments).reject(&:empty?)
+    return @values if segments.empty?
+    return @values if same_language?(@from)
+
+    provider = resolve_provider
+    from = resolve_source_language(provider, segments)
+    return @values if same_language?(from)
+
+    translated(document, passages, segments, provider, from)
+  end
+
+  private
+
+  # `from:` given means the whole pair is already known, so it is validated once, up front. `from:` nil means
+  # the target alone is checked before a possibly billed #detect runs, and the full pair only once it answers.
+  def resolve_source_language(provider, segments)
+    return @from.tap { |from| ensure_supported!(provider, from) } unless @from.nil?
+
+    ensure_supported!(provider, nil)
+    source_language(provider, segments).tap { |from| ensure_supported!(provider, from) }
+  end
+
+  # The `translate` event wraps everything a call that reaches a provider does, and nothing an early return does.
+  def translated(document, passages, segments, provider, from)
+    values = TranslationDiff::Leaves.count(@values)
+    characters = segments.sum { |segment| segment.core.size }
+    payload = { call_id: call_id, from: from.to_s, to: @to.to_s, provider: provider.cache_key,
+                values: values, characters: characters }
+    instrument("translate", payload) do
+      fill(provider, segments, from)
+      rebuild(document, passages)
+    end
+  end
+
+  # Opaque and short: a correlation key for this call's own events, generated once, never derived from the text.
+  def call_id = @call_id ||= SecureRandom.hex(6)
+
+  # Resolved at first use, never in the constructor: a value with nothing to translate needs no provider at all.
+  def resolve_provider
+    TranslationDiff::Providers.resolve(@requested_provider, config).tap { |provider| log("provider #{provider.class}") }
+  end
+
+  # The strings walk and the map walk visit the same leaves in the same order, and the value itself was never touched.
+  def rebuild(document, passages)
+    rendered = passages.map(&:render)
+    document.map { rendered.shift }
+  end
+
+  # Detection is attempted only where it is declared: every provider inherits a #detect that raises.
+  def source_language(provider, segments)
+    return @from unless @from.nil?
+
+    ensure_detects_language!(provider, Error, "translating")
+    provider.detect(segments.first.core)
+  end
+
+  # The cache answers for what it has, the provider for the rest, and only what came back is written home.
+  def fill(provider, segments, from)
+    cache = cache_for(provider, from)
+    misses = cache.fill(segments)
+    id = call_id
+    instrument("cache", call_id: id, provider: provider.cache_key,
+                        hits: segments.size - misses.size, misses: misses.size)
+    TranslationDiff::Dispatcher.new(provider: provider, from: from, to: @to, options: @options,
+                                    config: config, call_id: id).dispatch(misses)
+    store(cache, misses, provider)
+  end
+
+  # A translation already paid for at the provider must reach the caller even if writing it back never does.
+  def store(cache, misses, provider)
+    cache.store(misses)
+  rescue StandardError => e
+    warn_log("cache write failed (#{e.class}), the translation is returned uncached")
+    instrument("cache_error", call_id: call_id, provider: provider.cache_key, error: e.class.to_s)
+  end
+end

data/lib/translation_diff/version.rb

@@ -0,0 +1,3 @@
+module TranslationDiff
+  VERSION = "1.0.0".freeze
+end

data/lib/translation_diff.rb

@@ -0,0 +1,93 @@
+require "cgi/escape"
+require "digest/md5"
+require "digest/sha2"
+require "forwardable"
+require "securerandom"
+require "stringio"
+
+require "ox"
+
+require "translation_diff/version"
+require "translation_diff/errors"
+require "translation_diff/redaction"
+require "translation_diff/capabilities"
+require "json"
+require "translation_diff/languages"
+require "translation_diff/languages/set"
+require "translation_diff/languages/refresh"
+require "translation_diff/translation/usage"
+require "translation_diff/translation/request"
+require "translation_diff/translation/response"
+require "translation_diff/registry"
+require "translation_diff/document"
+require "translation_diff/leaves"
+require "translation_diff/markup"
+require "translation_diff/segment"
+require "translation_diff/batch"
+require "translation_diff/fragment"
+require "translation_diff/passage"
+require "translation_diff/sentence_cache"
+require "translation_diff/configuration"
+require "translation_diff/configuration/provider_option_owners"
+
+require "translation_diff/provider"
+require "translation_diff/http_provider"
+require "translation_diff/providers"
+require "translation_diff/providers/null"
+
+require "translation_diff/providers/deepl"
+require "translation_diff/providers/google"
+require "translation_diff/providers/azure"
+require "translation_diff/providers/modernmt"
+require "translation_diff/providers/libretranslate"
+require "translation_diff/providers/amazon"
+
+require "translation_diff/segmenters"
+require "translation_diff/segmenters/simple"
+require "translation_diff/segmenters/pragmatic"
+require "translation_diff/stores"
+require "translation_diff/stores/memory"
+require "translation_diff/stores/redis"
+require "translation_diff/active_record"
+require "translation_diff/active_record/support"
+require "translation_diff/stores/active_record"
+require "translation_diff/rate_limiters"
+require "translation_diff/rate_limiters/redis"
+require "translation_diff/rate_limiters/active_record"
+require "translation_diff/instrumentation"
+require "translation_diff/call_preparation"
+require "translation_diff/dispatcher"
+require "translation_diff/translator"
+require "translation_diff/preview"
+require "translation_diff/previewer"
+require "translation_diff/context"
+
+# Only when a host application has already loaded Rails -- never required unconditionally, so a non-Rails
+# application never pays for it, and the gem's own suite exercises this same guarded require, not a shortcut.
+require "translation_diff/railtie" if defined?(Rails::Railtie)
+
+module TranslationDiff
+  class << self
+    def config = @config ||= Configuration.new
+
+    def configure = yield(config)
+
+    # Without this, one test's configuration leaks into every test that runs after it.
+    def reset! = @config = nil
+
+    # An isolated copy of the configuration with the same entry point, for per-tenant settings.
+    def context(&) = Context.new(config.copy.tap(&))
+
+    # `provider:`, `config:` and `assume_supported:` are reserved; every other keyword is forwarded to the provider.
+    def translate(values, from: nil, to: nil, provider: nil, assume_supported: false, **)
+      Translator.new(values, from: from, to: to, provider: provider, config: config,
+                             assume_supported: assume_supported, **).call
+    end
+
+    # Answers what `translate` would do to `values`, without calling a provider or writing anything.
+    def preview(values, from: nil, to: nil, provider: nil, assume_supported: false, **)
+      Previewer.new(values, from: from, to: to, provider: provider, config: config,
+                            assume_supported: assume_supported, **).call
+    end
+  end
+end

data/translation_diff.gemspec

@@ -0,0 +1,56 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "translation_diff/version"
+
+# rubocop:disable-next Metrics/BlockLength
+Gem::Specification.new do |spec|
+  spec.name          = "translation_diff"
+  spec.version       = TranslationDiff::VERSION
+  spec.authors       = ["Islam Gagiev"]
+  spec.email         = ["omniacinis@gmail.com"]
+
+  spec.summary = "A translation cache that sends only new or changed sentences to your provider."
+  spec.description = %(
+TranslationDiff extracts translatable text from HTML, splits it into
+sentences, and caches each sentence by content hash. Only the sentences
+missing from the cache are sent to whichever translation provider you plug
+in through a small provider contract, so re-translating a long text after a
+small edit costs the price of the edit, not the whole text.
+  )
+  spec.homepage = "https://github.com/Halvanhelv/translation_diff"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.4"
+
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = spec.homepage
+    spec.metadata["rubygems_mfa_required"] = "true"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+          "public gem pushes."
+  end
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "minitest", "~> 6.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rubocop", "~> 1.90"
+  spec.add_development_dependency "simplecov", "~> 1.2"
+
+  # `ox` walks HTML; `pragmatic_segmenter` backs the default sentence
+  # segmenter and has zero dependencies of its own; `faraday` and
+  # `faraday-retry` are the HTTP transport every REST provider inherits.
+  # Everything else -- the connection pool, and whatever backs the cache and
+  # the rate limiter -- is supplied by the application and duck typed, so it
+  # stays out of the gemspec.
+  spec.add_dependency "faraday", "~> 2.9"
+  spec.add_dependency "faraday-retry", "~> 2.2"
+  spec.add_dependency "ox", "~> 2.14"
+  spec.add_dependency "pragmatic_segmenter", "~> 0.3"
+end

metadata

@@ -0,0 +1,243 @@
+--- !ruby/object:Gem::Specification
+name: translation_diff
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Islam Gagiev
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.90'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.90'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: pragmatic_segmenter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: "\nTranslationDiff extracts translatable text from HTML, splits it into\nsentences,
+  and caches each sentence by content hash. Only the sentences\nmissing from the cache
+  are sent to whichever translation provider you plug\nin through a small provider
+  contract, so re-translating a long text after a\nsmall edit costs the price of the
+  edit, not the whole text.\n  "
+email:
+- omniacinis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/release.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- data/languages/azure.json
+- data/languages/deepl.json
+- data/languages/google.json
+- data/languages/modernmt.json
+- docs/caching.md
+- docs/configuration.md
+- docs/contracts.md
+- docs/development.md
+- docs/errors.md
+- docs/how-it-works.md
+- docs/instrumentation.md
+- docs/languages.md
+- docs/providers.md
+- docs/sql-cache.md
+- lib/generators/translation_diff/install_generator.rb
+- lib/generators/translation_diff/templates/create_translation_diff_tables.rb.erb
+- lib/translation_diff.rb
+- lib/translation_diff/active_record.rb
+- lib/translation_diff/active_record/support.rb
+- lib/translation_diff/batch.rb
+- lib/translation_diff/call_preparation.rb
+- lib/translation_diff/capabilities.rb
+- lib/translation_diff/configuration.rb
+- lib/translation_diff/configuration/cache_guard_options.rb
+- lib/translation_diff/configuration/cache_ttl_option.rb
+- lib/translation_diff/configuration/option_table.rb
+- lib/translation_diff/configuration/provider_option_owners.rb
+- lib/translation_diff/context.rb
+- lib/translation_diff/dispatcher.rb
+- lib/translation_diff/document.rb
+- lib/translation_diff/errors.rb
+- lib/translation_diff/fragment.rb
+- lib/translation_diff/http_provider.rb
+- lib/translation_diff/instrumentation.rb
+- lib/translation_diff/languages.rb
+- lib/translation_diff/languages/refresh.rb
+- lib/translation_diff/languages/set.rb
+- lib/translation_diff/leaves.rb
+- lib/translation_diff/markup.rb
+- lib/translation_diff/passage.rb
+- lib/translation_diff/preview.rb
+- lib/translation_diff/previewer.rb
+- lib/translation_diff/provider.rb
+- lib/translation_diff/providers.rb
+- lib/translation_diff/providers/amazon.rb
+- lib/translation_diff/providers/azure.rb
+- lib/translation_diff/providers/deepl.rb
+- lib/translation_diff/providers/google.rb
+- lib/translation_diff/providers/libretranslate.rb
+- lib/translation_diff/providers/modernmt.rb
+- lib/translation_diff/providers/null.rb
+- lib/translation_diff/railtie.rb
+- lib/translation_diff/rate_limiters.rb
+- lib/translation_diff/rate_limiters/active_record.rb
+- lib/translation_diff/rate_limiters/redis.rb
+- lib/translation_diff/redaction.rb
+- lib/translation_diff/registry.rb
+- lib/translation_diff/segment.rb
+- lib/translation_diff/segmenters.rb
+- lib/translation_diff/segmenters/pragmatic.rb
+- lib/translation_diff/segmenters/simple.rb
+- lib/translation_diff/sentence_cache.rb
+- lib/translation_diff/stores.rb
+- lib/translation_diff/stores/active_record.rb
+- lib/translation_diff/stores/memory.rb
+- lib/translation_diff/stores/redis.rb
+- lib/translation_diff/tasks/translation_diff.rake
+- lib/translation_diff/translation/request.rb
+- lib/translation_diff/translation/response.rb
+- lib/translation_diff/translation/usage.rb
+- lib/translation_diff/translator.rb
+- lib/translation_diff/version.rb
+- translation_diff.gemspec
+homepage: https://github.com/Halvanhelv/translation_diff
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/Halvanhelv/translation_diff
+  source_code_uri: https://github.com/Halvanhelv/translation_diff
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: A translation cache that sends only new or changed sentences to your provider.
+test_files: []