translation_diff 1.0.0

data/lib/translation_diff/segmenters/pragmatic.rb

@@ -0,0 +1,102 @@
+require "pragmatic_segmenter"
+
+# Default segmenter: scores 76/80 on the Golden Rules corpus vs Simple's 47/80 (golden_rules_test.rb).
+class TranslationDiff::Segmenters::Pragmatic
+  # Raised only if computed offsets violate their own postcondition, never by ordinary sentence rewriting.
+  class Error < TranslationDiff::Error; end
+
+  # Without a language, Russian mis-segments: it treats "Проф." as a full sentence and stops there.
+  DEFAULT_LANGUAGE = "en".freeze
+
+  # A lone "\n" is incidental source formatting, not a paragraph break; a run of two or more is left alone.
+  SINGLE_NEWLINE = /(?<!\n)\n(?!\n)/
+
+  def self.build(_config) = new
+
+  def split_offsets(text, language: nil)
+    return [0] unless split_candidate?(text)
+
+    shadow = shadow_newlines(text)
+    sentences = segment(shadow, language)
+    return [0] if sentences.size <= 1
+
+    offsets = recover_offsets(shadow, sentences)
+    assert_valid_offsets(text, offsets)
+    offsets
+  end
+
+  private
+
+  def split_candidate?(text)
+    text.is_a?(String) && !text.strip.empty?
+  end
+
+  # pragmatic_segmenter treats any single newline as a sentence boundary, confirmed false on wrapped prose.
+  def shadow_newlines(text)
+    text.gsub(SINGLE_NEWLINE, " ")
+  end
+
+  # DeepL sends codes like "EN-GB"; the lookup is case-sensitive, so without this "RU" falls through to Common.
+  def normalize_language(language)
+    code = language.to_s.downcase.split(/[-_]/, 2).first
+    return DEFAULT_LANGUAGE unless PragmaticSegmenter::Languages::LANGUAGE_CODES.key?(code)
+
+    code
+  end
+
+  def segment(shadow, language)
+    PragmaticSegmenter::Segmenter.new(text: shadow, language: normalize_language(language)).segment
+  end
+
+  # Pinned 0.3.24's cleaner rewrites text (collapses spaces, respaces "Ph.D."), so some sentences can't be located.
+  def recover_offsets(shadow, sentences)
+    starts, cursor, stopped_early = walk(shadow, sentences)
+
+    # The first located sentence's own start is never a split point; only the ones after it are.
+    offsets = [0] + starts.drop(1)
+
+    # Cursor is verified evidence, not a guess; guarded against both ways it could break the offsets invariant.
+    offsets << cursor if stopped_early && cursor < shadow.length && cursor > offsets.last
+
+    offsets
+  end
+
+  # Returns the starts of every sentence located, the cursor after the last one, and whether the walk stopped early.
+  def walk(shadow, sentences)
+    cursor = 0
+    starts = []
+
+    sentences.each do |sentence|
+      match = locate(shadow, sentence, cursor)
+      next if match == :skip
+      return [starts, cursor, true] if match.nil?
+
+      starts << match[0]
+      cursor = match[1]
+    end
+
+    [starts, cursor, false]
+  end
+
+  # :skip for empty (an empty match would never advance the cursor); nil if a non-empty sentence isn't found.
+  def locate(shadow, sentence, cursor)
+    return :skip if sentence.empty?
+
+    start = shadow.index(sentence, cursor)
+    return nil if start.nil?
+
+    [start, start + sentence.length]
+  end
+
+  # Asserts the postcondition rather than trusting construction forever: a wrong offset would silently corrupt.
+  def assert_valid_offsets(text, offsets)
+    return if offsets.first.zero? &&
+              offsets.each_cons(2).all? { |a, b| a < b } &&
+              offsets.all? { |offset| offset < text.length }
+
+    raise Error, "computed offsets #{offsets.inspect} do not start at 0, strictly increase, and stay " \
+                 "within the text -- refusing to hand them back. text: #{text.inspect}"
+  end
+end
+
+TranslationDiff::Segmenters.registry.register(:pragmatic, TranslationDiff::Segmenters::Pragmatic)

data/lib/translation_diff/segmenters/simple.rb

@@ -0,0 +1,122 @@
+# No runtime dependency, but deliberately conservative: every guard exists to turn a split off, never on.
+class TranslationDiff::Segmenters::Simple
+  # CJK terminators need no trailing whitespace and no guards -- those scripts have no case or abbreviations.
+  TERMINATOR = /[.!?…]+|[。！？]+/
+
+  CJK_TERMINATOR = /\A[。！？]/
+
+  # Words that end in "." without ending a sentence, matched case-insensitively; extend for your own domain.
+  ABBREVIATIONS = %w[
+    т.е. т.д. т.п. см. рис. стр. гр. ул. г. руб. проф. акад. тыс. млн. млрд. им.
+    Mr. Mrs. Ms. Dr. Prof. St. etc. e.g. i.e. vs. approx. No. im. fig. Fig. vol. p. pp.
+  ].map(&:downcase).freeze
+
+  # Strips leading punctuation like an opening quote so `"Dr. Smith` still guards on "Dr.".
+  WORD_TAIL = /[\p{L}\p{N}.]+\z/
+
+  def self.build(_config) = new
+
+  # language: is part of the shared segmenter contract but ignored here -- these rules are language-neutral.
+  # rubocop:disable-next Lint/UnusedMethodArgument
+  def split_offsets(text, language: nil)
+    offsets = [0]
+    position = 0
+
+    while (match = TERMINATOR.match(text, position))
+      boundary = boundary_for(text, match)
+      offsets << boundary if boundary
+      position = match.end(0)
+    end
+
+    offsets
+  end
+
+  private
+
+  def boundary_for(text, match)
+    if CJK_TERMINATOR.match?(match[0])
+      cjk_boundary(text, match.end(0))
+    else
+      latin_boundary(text, match)
+    end
+  end
+
+  # If whitespace follows, it's attached to the sentence that just ended, not left as a leading gap.
+  def cjk_boundary(text, run_end)
+    boundary = skip_whitespace(text, run_end)
+    boundary if boundary < text.length
+  end
+
+  # A bare "." with nothing after it is not a sentence break, it is a string that stops mid-thought.
+  def latin_boundary(text, match)
+    run_end = match.end(0)
+    return unless whitespace?(text[run_end])
+
+    boundary = skip_whitespace(text, run_end)
+    return if boundary >= text.length
+    return if guarded?(text, match.begin(0), match[0], boundary)
+
+    boundary
+  end
+
+  def guarded?(text, run_start, run, next_index)
+    word_before = word_before(text, run_start)
+    next_char = text[next_index]
+
+    lowercase_follows?(next_char) ||
+      abbreviation?(word_before, run) ||
+      single_letter?(word_before, run) ||
+      digits_on_both_sides?(word_before, next_char) ||
+      url_or_email?(word_before, run)
+  end
+
+  def lowercase_follows?(char)
+    !char.nil? && char.match?(/\p{Ll}/)
+  end
+
+  def abbreviation?(word_before, run)
+    return false unless run.start_with?(".")
+
+    ABBREVIATIONS.include?("#{word_tail(word_before)}.".downcase)
+  end
+
+  def single_letter?(word_before, run)
+    return false unless run.start_with?(".")
+
+    tail = word_tail(word_before)
+    tail.length == 1 && tail.match?(/\p{L}/)
+  end
+
+  def digits_on_both_sides?(word_before, next_char)
+    return false if next_char.nil?
+
+    word_before[-1]&.match?(/\d/) && next_char.match?(/\d/)
+  end
+
+  def url_or_email?(word_before, run)
+    token = "#{word_before}#{run}"
+    token.include?("://") || token.include?("@")
+  end
+
+  # The maximal run of non-whitespace characters immediately before the terminator.
+  def word_before(text, run_start)
+    start = run_start
+    start -= 1 while start.positive? && !whitespace?(text[start - 1])
+    text[start...run_start]
+  end
+
+  def word_tail(word_before)
+    word_before[WORD_TAIL] || ""
+  end
+
+  def skip_whitespace(text, index)
+    index += 1 while index < text.length && whitespace?(text[index])
+    index
+  end
+
+  def whitespace?(char)
+    !char.nil? && char.match?(/\s/)
+  end
+end
+
+TranslationDiff::Segmenters.registry.register(:simple, TranslationDiff::Segmenters::Simple)

data/lib/translation_diff/segmenters.rb

@@ -0,0 +1,4 @@
+module TranslationDiff::Segmenters
+  # Segmenters, by name. `config.segmenter = :simple` resolves through here.
+  def self.registry = @registry ||= TranslationDiff::Registry.new("segmenter")
+end

data/lib/translation_diff/sentence_cache.rb

@@ -0,0 +1,76 @@
+# Reads and writes sentence translations under the pipeline's cache key format, without touching the caller's array.
+class TranslationDiff::SentenceCache
+  # Its own class, so rescuing an unusable option cannot also swallow a store or provider failure.
+  class Error < TranslationDiff::Error; end
+
+  # The value types the key format can render. An allowlist: what it cannot render must raise, not be guessed at.
+  SCALARS = [String, Symbol, Numeric, TrueClass, FalseClass, NilClass].freeze
+
+  def initialize(store:, provider:, from:, to:, options: {})
+    @store = store
+    @provider = provider
+    @from = from.downcase
+    @to = to.downcase
+    @options = options
+  end
+
+  # provider:from:to:sentence, with a digest of the per-call options wedged in as a field of its own when there are any.
+  def key(segment)
+    [@provider, @from, @to, *options_digest, Digest::MD5.hexdigest(segment.body)].join(":")
+  end
+
+  # Sets a translation on every segment the store already has cached, and hands back the rest as a new array.
+  def fill(segments)
+    values = @store.read_multi(segments.map { |segment| key(segment) })
+    misses = []
+    segments.each_with_index do |segment, index|
+      value = values[index]
+      value.nil? ? misses << segment : segment.translation = value
+    end
+    misses
+  end
+
+  # Writes back only the segments that carry a translation; an untranslated segment has nothing worth caching.
+  # A store that batches gets one call; one that does not keeps the per-key contract it was written against.
+  def store(segments)
+    translated = segments.select(&:translated?)
+    return translated if translated.empty?
+    return translated.each { |segment| @store.write(key(segment), segment.translation) } if legacy_store?
+
+    @store.write_multi(translated.map { |segment| [key(segment), segment.translation] })
+    translated
+  end
+
+  private
+
+  # A store written against the write-only contract, before write_multi existed, cannot be handed a batch.
+  def legacy_store? = !@store.respond_to?(:write_multi)
+
+  # No options contributes no field at all, which is the four-field key every already-warm cache is keyed on.
+  def options_digest
+    return [] if @options.empty?
+
+    [Digest::MD5.hexdigest(canonical_options)[0, 8]]
+  end
+
+  # Sorted on the name's string form, because a Symbol and a String key are not comparable with each other.
+  def canonical_options
+    @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{canonical(name, value)}" }.join(",")
+  end
+
+  # A Hash canonicalises the way the options hash itself does, an Array its elements in order, joined the same way.
+  def canonical(name, value)
+    case value
+    when Hash then value.sort_by { |k, _| k.to_s }.map { |k, v| "#{k}=#{canonical(name, v)}" }.join(",")
+    when Array then value.map { |element| canonical(name, element) }.join(",")
+    else scalar(name, value)
+    end
+  end
+
+  # A key that is silently wrong costs a caller their whole cache and tells them nothing, so refuse to build one.
+  def scalar(name, value)
+    return value.inspect if SCALARS.any? { |type| value.is_a?(type) }
+
+    raise Error, "cache option #{name} (a #{value.class}) has no stable string form"
+  end
+end

data/lib/translation_diff/stores/active_record.rb

@@ -0,0 +1,106 @@
+# Caches translations in the application's own database; ActiveRecord is required on first use, never at load.
+class TranslationDiff::Stores::ActiveRecord
+  include TranslationDiff::ActiveRecord::Support
+
+  def self.build(config)
+    new(namespace: config.cache_namespace, ttl: config.cache_ttl,
+        table_name: config.cache_table_name, base: config.active_record_base,
+        prune_probability: config.cache_prune_probability)
+  end
+
+  def initialize(namespace:, ttl:, table_name:, base: nil, prune_probability: 0.0)
+    @namespace = namespace
+    @ttl = ttl
+    @table_name = table_name
+    @base = base
+    @prune_probability = prune_probability
+  end
+
+  # One query, then the caller's order restored -- a missing or expired key is a nil in its own position.
+  def read_multi(keys)
+    return [] if keys.empty?
+
+    digests = keys.map { |key| digest(key) }
+    found = live.where(key_digest: digests).pluck(:key_digest, :translation).to_h
+    digests.map { |d| found[d] }
+  end
+
+  def write(key, value)
+    write_multi([[key, value]])
+    value
+  end
+
+  # One upsert for the whole batch; the unique index makes the second write of a key replace the first.
+  def write_multi(pairs)
+    return pairs if pairs.empty?
+
+    # A savepoint, not the caller's own transaction: a failed write must not abort a transaction it does not own.
+    model.transaction(requires_new: true) do
+      model.upsert_all(pairs.to_h.map { |key, value| row(key, value) }, **upsert_options(model.connection))
+    end
+    prune_sometimes
+    pairs
+  rescue StandardError => e
+    raise unless ar_error?(e)
+
+    raise redacted_error(e), cause: nil
+  end
+
+  # Reads never serve an expired row; deleting one is this, and it is the host's call when to run it.
+  def prune = model.where(namespace: @namespace).where(expires_at: ...Time.now.utc).delete_all
+
+  private
+
+  # MySQL's adapter never answers true here and its ON DUPLICATE KEY UPDATE already targets every unique key.
+  def upsert_options(connection)
+    options = { record_timestamps: true }
+    options[:unique_by] = %i[namespace key_digest] if connection.supports_insert_conflict_target?
+    options
+  end
+
+  # upsert_all inlines values into the statement it sends, so the adapter's own message can carry a whole row --
+  # this names the adapter's error class and the statement's shape, never the row a caller's logger already has.
+  def redacted_error(error)
+    adapter_error = error.cause&.class || error.class
+    TranslationDiff::Error.new("the cache write failed (#{adapter_error}): an upsert into " \
+                               "#{@table_name}(namespace, key_digest, translation, expires_at)")
+  end
+
+  # Its own message, naming the statement prune actually runs -- a failed prune is not a failed upsert.
+  def redacted_prune_error(error)
+    adapter_error = error.cause&.class || error.class
+    TranslationDiff::Error.new("the cache prune failed (#{adapter_error}): a delete from " \
+                               "#{@table_name}(namespace, expires_at)")
+  end
+
+  def row(key, value)
+    { namespace: @namespace, key_digest: digest(key), translation: value, expires_at: expires_at }
+  end
+
+  def expires_at = @ttl.nil? ? nil : Time.now.utc + @ttl
+
+  # SHA256 hex is 64 characters whatever the key was, which is what makes the unique index portable.
+  def digest(key) = Digest::SHA256.hexdigest(key.to_s)
+
+  def live
+    model.where(namespace: @namespace)
+         .where(expires_at: nil).or(model.where(namespace: @namespace).where(expires_at: Time.now.utc...))
+  end
+
+  # Its own savepoint, not write's -- a failing prune must not poison a transaction the caller owns either.
+  def prune_sometimes
+    return unless @prune_probability.positive? && rand < @prune_probability
+
+    model.transaction(requires_new: true) { prune }
+  rescue StandardError => e
+    raise unless ar_error?(e)
+
+    raise redacted_prune_error(e), cause: nil
+  end
+
+  def active_record_feature = "the cache"
+  def active_record_component = "ActiveRecord cache store"
+  def active_record_upsert_detail = "upsert_all takes unique_by and record_timestamps there."
+end
+
+TranslationDiff::Stores.register(:active_record, TranslationDiff::Stores::ActiveRecord)

data/lib/translation_diff/stores/memory.rb

@@ -0,0 +1,34 @@
+# The default cache, a bounded in-process LRU. NOT thread-safe, deliberately -- set `redis_url` for that.
+class TranslationDiff::Stores::Memory
+  def self.build(config) = new(max_size: config.cache_max_size)
+
+  def initialize(max_size:)
+    @max_size = max_size
+    @entries = {}
+  end
+
+  def read_multi(keys)
+    keys.map { |key| touch(key) }
+  end
+
+  def write(key, value)
+    @entries.delete(key)
+    @entries[key] = value
+    @entries.shift while @entries.size > @max_size
+    value
+  end
+
+  def write_multi(pairs)
+    pairs.each { |key, value| write(key, value) }
+  end
+
+  private
+
+  def touch(key)
+    return nil unless @entries.key?(key)
+
+    @entries[key] = @entries.delete(key)
+  end
+end
+
+TranslationDiff::Stores.register(:memory, TranslationDiff::Stores::Memory)

data/lib/translation_diff/stores/redis.rb

@@ -0,0 +1,49 @@
+class TranslationDiff::Stores::Redis
+  ONE_WEEK = 60 * 60 * 24 * 7
+  DEFAULT_NAMESPACE = "translation-diff".freeze
+
+  def self.build(config)
+    new(config.redis_pool, timeout: config.cache_ttl, namespace: config.cache_namespace)
+  end
+
+  # `connection_pool` is duck-typed to #with; neither connection_pool nor redis-namespace is a hard dependency.
+  def initialize(connection_pool, timeout: ONE_WEEK, namespace: DEFAULT_NAMESPACE)
+    @connection_pool = connection_pool
+    @timeout = timeout
+    @namespace = namespace
+  end
+
+  def read_multi(keys)
+    redis { |redis| redis.mget(*keys) }
+  end
+
+  # A non-positive or nil timeout means never expires, the same rule the SQL store applies to cache_ttl.
+  def write(key, value)
+    redis { |redis| write_one(redis, key, value) }
+  end
+
+  def write_multi(pairs)
+    return pairs if pairs.empty?
+
+    redis { |redis| redis.pipelined { |p| pairs.each { |key, value| write_one(p, key, value) } } }
+    pairs
+  end
+
+  private
+
+  attr_reader :connection_pool, :timeout, :namespace
+
+  def redis
+    connection_pool.with do |redis|
+      yield ::Redis::Namespace.new(namespace, redis: redis)
+    end
+  end
+
+  def write_one(redis, key, value)
+    expiring? ? redis.setex(key, timeout, value) : redis.set(key, value)
+  end
+
+  def expiring? = timeout.is_a?(Numeric) && timeout.positive?
+end
+
+TranslationDiff::Stores.register(:redis, TranslationDiff::Stores::Redis)

data/lib/translation_diff/stores.rb

@@ -0,0 +1,9 @@
+# Cache stores, by name; assigning an object to `config.cache` bypasses this entirely.
+module TranslationDiff::Stores
+  def self.register(name, klass) = registry.register(name, klass)
+  def self.build(name, config) = registry.build(name, config)
+  def self.registered?(name) = registry.registered?(name)
+  def self.names = registry.names
+  def self.classes = registry.classes
+  def self.registry = @registry ||= TranslationDiff::Registry.new("cache store")
+end

data/lib/translation_diff/tasks/translation_diff.rake

@@ -0,0 +1,21 @@
+# Shipped in the gem so a host application's own `rake` sees it -- a dependency's Rakefile is never loaded.
+namespace :translation_diff do
+  desc "Delete expired rows from the SQL cache and rate-limit tables"
+  task :prune do
+    require "translation_diff"
+
+    cache_store = TranslationDiff.config.cache_store
+    if cache_store.respond_to?(:prune)
+      puts "pruned #{cache_store.prune} expired cache rows"
+    else
+      puts "the configured cache store (#{cache_store.class}) does not support pruning"
+    end
+
+    rate_limiter = TranslationDiff.config.rate_limiter_instance
+    if rate_limiter.respond_to?(:prune)
+      puts "pruned #{rate_limiter.prune} expired rate-limit rows"
+    else
+      puts "the configured rate limiter (#{rate_limiter.class}) does not support pruning"
+    end
+  end
+end

data/lib/translation_diff/translation/request.rb

@@ -0,0 +1,8 @@
+# What the pipeline asks a provider for; `from` nil means the provider should detect the source language.
+module TranslationDiff::Translation
+  Request = Data.define(:texts, :from, :to, :options) do
+    def initialize(texts:, from:, to:, options: {})
+      super
+    end
+  end
+end

data/lib/translation_diff/translation/response.rb

@@ -0,0 +1,35 @@
+# ::build is the one guard over every provider's parse step: a wrong count or a non-String would surface far away.
+module TranslationDiff::Translation
+  Response = Data.define(:texts, :detected_source, :usage) do
+    def self.build(request:, texts:, detected_source: nil, usage: nil)
+      ensure_count!(request, texts)
+      ensure_strings!(texts)
+
+      # Every provider's text lands here, so it takes the same path @core did: escaped, then decoded once, so an
+      # entity a provider genuinely sent survives as the entity it is rather than the bare character it decodes to.
+      new(texts: texts.map { |text| decoded(text) }, detected_source: detected_source, usage: usage)
+    end
+
+    def self.decoded(text)
+      TranslationDiff::Markup.decode_entities(TranslationDiff::Markup.escape_bare_angles(text))
+    end
+
+    def self.ensure_count!(request, texts)
+      return if texts.size == request.texts.size
+
+      raise TranslationDiff::ResponseError,
+            "Provider returned #{texts.size} translations for #{request.texts.size} values"
+    end
+
+    # The class, never the value: the value is either the customer's text or the provider's own error object.
+    def self.ensure_strings!(texts)
+      index = texts.index { |text| !text.is_a?(String) }
+      return if index.nil?
+
+      raise TranslationDiff::ResponseError,
+            "Provider returned #{texts[index].class} rather than a translation at position " \
+            "#{index} of #{texts.size}. A response can be well-formed and still carry no " \
+            "translation for one input -- a per-string failure inside a batch that answered 200."
+    end
+  end
+end

data/lib/translation_diff/translation/usage.rb

@@ -0,0 +1,8 @@
+# `billed_characters` is nil, not 0, when a provider doesn't report it; `tokens`/`model` exist for LLM providers.
+module TranslationDiff::Translation
+  Usage = Data.define(:characters, :billed_characters, :tokens, :model) do
+    def initialize(characters:, billed_characters: nil, tokens: nil, model: nil)
+      super
+    end
+  end
+end