deepl_diff 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca22132db1cfc83ab8e56a9a17bd635ca3648e9d7fde32a2df13dce21991f946
-  data.tar.gz: '018f598439a77db2e229d2c95f7b8ff0c14c0f17ba7130e19cbdf3f897b716a6'
+  metadata.gz: 83f226ab695b30cfc248762a7760bceb6bf9166d9a6f7a9bee620235309c33d2
+  data.tar.gz: cb0b0e175dce7ae19ba282fb86a8cc15fac1266fc373ee1b5c56dd0cac707fba
 SHA512:
-  metadata.gz: 0144f0c5d615c25036d0f2f2b41238667e6ab045442714e45235c8192f49c70fdeeb92753c6e7ec5f0e1189de3064191bdbb2014108a5715942ee1455f6ae76f
-  data.tar.gz: 71a0a99b3f5869bec1f2383f8d212c6ead9bd63f9e5eb5a9f8ad98ed2aa93f0701d5775b9de3764c21aad7ac96f11afe2a506e674056f9a97a80ed5d357c65b6
+  metadata.gz: 0bd5a40578adf7e42ff6bae724f6983bae992af787bf4e0ade831353259683e097fc1a1f34cb25c3ac7a563faf6a75d6cf9e8983d7d2e973dbf9ec97a5897b30
+  data.tar.gz: b7abc9af4e0ee1f33c5dd3dafc764069776c7668a2c361bd74064c6068f4be3e98e399abedcb216d57dcb8e08e8010e89db487d3bf79f849962e6f5c08fe057a

data/deepl_diff.gemspec

@@ -33,6 +33,15 @@ between revisions of long texts.
           "public gem pushes."
   end
 
+  spec.post_install_message = <<~MESSAGE
+    deepl_diff is being replaced by translation_diff, which supports
+    translation providers other than DeepL.
+
+    This version of deepl_diff keeps working and will not change again.
+
+    https://rubygems.org/gems/translation_diff
+  MESSAGE
+
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end

data/lib/deepl_diff/chunker.rb

@@ -3,7 +3,7 @@
 class DeepLDiff::Chunker
   class Error < StandardError; end
 
-  Chunk = Struct.new(:texts, :bytesize)
+  Chunk = Struct.new(:texts, :escaped_size)
 
   MAX_CHUNK_SIZE = 1700
   COUNT_LIMIT = 300
@@ -39,20 +39,25 @@ class DeepLDiff::Chunker
 
   def next_chunk?(tail, value)
     tail.nil? ||
-      (size(value) + tail.bytesize > limit) ||
-      tail.texts.size > count_limit
+      (escaped_size(value) + tail.escaped_size > limit) ||
+      tail.texts.size >= count_limit
   end
 
-  def size(text)
+  # What the limit is about is the size of the request that goes over the
+  # wire, so every measurement here is of the escaped form. Mixing it with
+  # String#size lets a chunk of non-ASCII text run several times over.
+  def escaped_size(text)
     CGI.escape(text).size
   end
 
   def update_chunk(chunk, value)
     chunk.texts << value
-    chunk.bytesize += value.size
+    chunk.escaped_size += escaped_size(value)
   end
 
   def validate_value_size(value)
-    raise Error, "Too long part #{value.size} > #{limit}" if value.size > limit
+    size = escaped_size(value)
+
+    raise Error, "Too long part #{size} > #{limit}" if size > limit
   end
 end

data/lib/deepl_diff/request.rb

@@ -3,18 +3,22 @@
 class DeepLDiff::Request
   extend Forwardable
 
+  class Error < StandardError; end
+
   def_delegators :DeepLDiff, :api, :cache_store, :rate_limiter
   def_delegators :"DeepLDiff::Linearizer", :linearize, :restore
 
   def initialize(values, options)
     @values = values
-    @options = options
+    # #from and #to consume their keys so the rest can go to the API as-is.
+    # Copy first: the caller's hash is theirs, and it is often frozen.
+    @options = options.dup
   end
 
   def call
     validate_globals
 
-    return values if from == to || values.empty?
+    return values if same_language? || nothing_to_translate?
 
     translation
   end
@@ -31,6 +35,19 @@ class DeepLDiff::Request
     @to ||= options.delete(:to) { nil }
   end
 
+  # A detected language arrives as a String while :to is usually a Symbol, so
+  # the two have to be compared on equal footing or the short circuit never
+  # fires and the text gets translated into its own language.
+  def same_language?
+    !to.nil? && from.to_s.casecmp?(to.to_s)
+  end
+
+  # Covers values holding no translatable text at all: "", nil, an empty
+  # collection, or a scalar the tokenizer has nothing to say about.
+  def nothing_to_translate?
+    text_tokens_texts.all?(&:empty?)
+  end
+
   def detect_language
     api.translate(text_tokens_texts.join(" ")[0..100], nil, to)
        .detected_source_language.downcase
@@ -126,7 +143,12 @@ class DeepLDiff::Request
   # Restores texts from tokens
   # [..., "<b>Horoshiy</b> Malchik", ...]
   def texts_translated
-    @texts_translated ||= tokens_translated.map do |group|
+    @texts_translated ||= tokens_translated.map.with_index do |group, index|
+      source = texts[index]
+      # Only strings are rebuilt from tokens. Anything else has no tokens to
+      # rebuild from; nil keeps collapsing to "" the way it always has.
+      next source unless source.nil? || source.is_a?(String)
+
       group.map { |value, type| type == :text ? value : fix_ascii(value) }.join
     end
   end
@@ -138,7 +160,13 @@ class DeepLDiff::Request
 
   def call_api(values)
     check_rate_limit(values)
-    [api.translate(values, from, to, options)].flatten.map(&:text)
+    translations = [api.translate(values, from, to, options)].flatten.map(&:text)
+    return translations if translations.size == values.size
+
+    # Letting a short response through means shifting nils into the results,
+    # which surfaces much later as a NoMethodError far from the cause.
+    raise Error,
+          "API returned #{translations.size} translations for #{values.size} values"
   end
 
   def cache

data/lib/deepl_diff/tokenizer.rb

@@ -149,7 +149,8 @@ class DeepLDiff::Tokenizer < Ox::Sax
 
   class << self
     def tokenize(value)
-      return [] if value.nil?
+      # Anything that is not a string has no markup and no sentences in it.
+      return [] unless value.is_a?(String)
 
       tokenizer = new(value).tap do |h|
         Ox.sax_parse(h, StringIO.new(value), HTML_OPTIONS)

data/lib/deepl_diff/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeepLDiff
-  VERSION = "2.0.0"
+  VERSION = "2.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deepl_diff
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Islam Gagiev
@@ -127,6 +127,13 @@ metadata:
   homepage_uri: https://github.com/Halvanhelv/deepl_diff
   source_code_uri: https://github.com/Halvanhelv/deepl_diff
   rubygems_mfa_required: 'true'
+post_install_message: |
+  deepl_diff is being replaced by translation_diff, which supports
+  translation providers other than DeepL.
+
+  This version of deepl_diff keeps working and will not change again.
+
+  https://rubygems.org/gems/translation_diff
 rdoc_options: []
 require_paths:
 - lib