deepl_diff 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca22132db1cfc83ab8e56a9a17bd635ca3648e9d7fde32a2df13dce21991f946
-  data.tar.gz: '018f598439a77db2e229d2c95f7b8ff0c14c0f17ba7130e19cbdf3f897b716a6'
+  metadata.gz: 2c71f2ab36a0f279252d4a42bdd0fc915b51668c87ab05279474598487d19e74
+  data.tar.gz: bf24b7de677ac350e1fbe174409442626ff45101a57a04909494e46f36c0a5d5
 SHA512:
-  metadata.gz: 0144f0c5d615c25036d0f2f2b41238667e6ab045442714e45235c8192f49c70fdeeb92753c6e7ec5f0e1189de3064191bdbb2014108a5715942ee1455f6ae76f
-  data.tar.gz: 71a0a99b3f5869bec1f2383f8d212c6ead9bd63f9e5eb5a9f8ad98ed2aa93f0701d5775b9de3764c21aad7ac96f11afe2a506e674056f9a97a80ed5d357c65b6
+  metadata.gz: 14e9475cad58ca2114b1849a442b2f01cbb958a2413bd86603c29fddc07cb5d815e8bbb342608b77dc353b1a7899f15b6d447a36a30cd0ef9c591aa726c6dd3d
+  data.tar.gz: 3fb543ab7c6cd3111bcd0ddf4843cc1b8d8ec6e4f0f35b18c8c85fda2be590641e2bd2ff3679a3150ca8dd3f36bcd5db47536a1bc1675ec5c6fcc6f7ac84010a

data/lib/deepl_diff/chunker.rb

@@ -3,7 +3,7 @@
 class DeepLDiff::Chunker
   class Error < StandardError; end
 
-  Chunk = Struct.new(:texts, :bytesize)
+  Chunk = Struct.new(:texts, :escaped_size)
 
   MAX_CHUNK_SIZE = 1700
   COUNT_LIMIT = 300
@@ -39,20 +39,25 @@ class DeepLDiff::Chunker
 
   def next_chunk?(tail, value)
     tail.nil? ||
-      (size(value) + tail.bytesize > limit) ||
-      tail.texts.size > count_limit
+      (escaped_size(value) + tail.escaped_size > limit) ||
+      tail.texts.size >= count_limit
   end
 
-  def size(text)
+  # What the limit is about is the size of the request that goes over the
+  # wire, so every measurement here is of the escaped form. Mixing it with
+  # String#size lets a chunk of non-ASCII text run several times over.
+  def escaped_size(text)
     CGI.escape(text).size
   end
 
   def update_chunk(chunk, value)
     chunk.texts << value
-    chunk.bytesize += value.size
+    chunk.escaped_size += escaped_size(value)
   end
 
   def validate_value_size(value)
-    raise Error, "Too long part #{value.size} > #{limit}" if value.size > limit
+    size = escaped_size(value)
+
+    raise Error, "Too long part #{size} > #{limit}" if size > limit
   end
 end

data/lib/deepl_diff/request.rb

@@ -3,18 +3,22 @@
 class DeepLDiff::Request
   extend Forwardable
 
+  class Error < StandardError; end
+
   def_delegators :DeepLDiff, :api, :cache_store, :rate_limiter
   def_delegators :"DeepLDiff::Linearizer", :linearize, :restore
 
   def initialize(values, options)
     @values = values
-    @options = options
+    # #from and #to consume their keys so the rest can go to the API as-is.
+    # Copy first: the caller's hash is theirs, and it is often frozen.
+    @options = options.dup
   end
 
   def call
     validate_globals
 
-    return values if from == to || values.empty?
+    return values if same_language? || nothing_to_translate?
 
     translation
   end
@@ -31,6 +35,19 @@ class DeepLDiff::Request
     @to ||= options.delete(:to) { nil }
   end
 
+  # A detected language arrives as a String while :to is usually a Symbol, so
+  # the two have to be compared on equal footing or the short circuit never
+  # fires and the text gets translated into its own language.
+  def same_language?
+    !to.nil? && from.to_s.casecmp?(to.to_s)
+  end
+
+  # Covers values holding no translatable text at all: "", nil, an empty
+  # collection, or a scalar the tokenizer has nothing to say about.
+  def nothing_to_translate?
+    text_tokens_texts.all?(&:empty?)
+  end
+
   def detect_language
     api.translate(text_tokens_texts.join(" ")[0..100], nil, to)
        .detected_source_language.downcase
@@ -126,7 +143,12 @@ class DeepLDiff::Request
   # Restores texts from tokens
   # [..., "<b>Horoshiy</b> Malchik", ...]
   def texts_translated
-    @texts_translated ||= tokens_translated.map do |group|
+    @texts_translated ||= tokens_translated.map.with_index do |group, index|
+      source = texts[index]
+      # Only strings are rebuilt from tokens. Anything else has no tokens to
+      # rebuild from; nil keeps collapsing to "" the way it always has.
+      next source unless source.nil? || source.is_a?(String)
+
       group.map { |value, type| type == :text ? value : fix_ascii(value) }.join
     end
   end
@@ -138,7 +160,13 @@ class DeepLDiff::Request
 
   def call_api(values)
     check_rate_limit(values)
-    [api.translate(values, from, to, options)].flatten.map(&:text)
+    translations = [api.translate(values, from, to, options)].flatten.map(&:text)
+    return translations if translations.size == values.size
+
+    # Letting a short response through means shifting nils into the results,
+    # which surfaces much later as a NoMethodError far from the cause.
+    raise Error,
+          "API returned #{translations.size} translations for #{values.size} values"
   end
 
   def cache

data/lib/deepl_diff/tokenizer.rb

@@ -149,7 +149,8 @@ class DeepLDiff::Tokenizer < Ox::Sax
 
   class << self
     def tokenize(value)
-      return [] if value.nil?
+      # Anything that is not a string has no markup and no sentences in it.
+      return [] unless value.is_a?(String)
 
       tokenizer = new(value).tap do |h|
         Ox.sax_parse(h, StringIO.new(value), HTML_OPTIONS)

data/lib/deepl_diff/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeepLDiff
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deepl_diff
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Islam Gagiev