deepl_diff 1.1.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e41201f9e6bf36b1fa7b9d34b58d3aa6822550d293c5ee4dd2a20a0edd75a887
-  data.tar.gz: 7d7dece832fa86e2d1e1f8b9c2c7083bd0f3a17cbd7101671cd44854cd6ac6d4
+  metadata.gz: 2c71f2ab36a0f279252d4a42bdd0fc915b51668c87ab05279474598487d19e74
+  data.tar.gz: bf24b7de677ac350e1fbe174409442626ff45101a57a04909494e46f36c0a5d5
 SHA512:
-  metadata.gz: 68defc006c6803311d571a96b0f45f5eed120c55ecf0ad854fd4303c8a780f447b190c20bd48cc3484fd44ddecf18ae1b4b22691b4a14cc054a6154d410523eb
-  data.tar.gz: a455a6e767ae6c6afe390db2720fbbe9adebe096f3bbf6f0fe735ad1a54408e54ee4cbbed5640bcdad3dcf92705fed4187c8ddb4cabed834c3d0434294093cce
+  metadata.gz: 14e9475cad58ca2114b1849a442b2f01cbb958a2413bd86603c29fddc07cb5d815e8bbb342608b77dc353b1a7899f15b6d447a36a30cd0ef9c591aa726c6dd3d
+  data.tar.gz: 3fb543ab7c6cd3111bcd0ddf4843cc1b8d8ec6e4f0f35b18c8c85fda2be590641e2bd2ff3679a3150ca8dd3f36bcd5db47536a1bc1675ec5c6fcc6f7ac84010a

data/.rubocop.yml

@@ -24,5 +24,6 @@ Gemspec/DevelopmentDependencies:
 Metrics/ClassLength:
   Exclude:
     - lib/deepl_diff/tokenizer.rb
+    - lib/deepl_diff/request.rb
     # Test classes are mostly tables of cases.
     - test/**/*

data/README.md

@@ -17,6 +17,16 @@ If your user changes a single word within the long description, you will be char
 
 Much better approach is to try to translate every repeated structural element (sentence) in your texts array just once to save money. This gem helps to make it done.
 
+## Dependencies
+
+This gem loads two: [`ox`](https://github.com/ohler55/ox) to walk the HTML, and
+[`punkt-segmenter`](https://github.com/lfcipriani/punkt-segmenter) to split text
+into sentences.
+
+Everything else is duck typed and supplied by you: `DeepLDiff.api` is anything
+answering to `#translate`, and both `RedisCacheStore` and `RedisRateLimiter`
+take anything answering to `#with`. Bring your own client, pool and store.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -38,11 +48,14 @@ Or install it yourself as:
 ```ruby
 require "deepl_diff"
 
-# This dependencies are not included, as you might need to roll your own cache based on different store
+# None of these are dependencies of this gem. It loads only `ox` and
+# `punkt-segmenter`; the API client, the connection pool, and whatever backs
+# the cache and the rate limiter are yours to choose and to require.
+require "deepl"
 require "redis"
 require "connection_pool"
 require "redis-namespace"
-require "ratelimit" # Optional, if you will use
+require "ratelimit" # Optional, only if you use the rate limiter
 
 # Setup https://github.com/wikiti/deepl-rb
 DeepL.configure do |config|
@@ -108,10 +121,10 @@ DeepL API has a limitation: query can not be longer than approximately 128 KB. I
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/wikiti/deepl-rb.
+Bug reports and pull requests are welcome on GitHub at https://github.com/Halvanhelv/deepl_diff.

data/deepl_diff.gemspec

@@ -45,12 +45,9 @@ between revisions of long texts.
   spec.add_development_dependency "rubocop", "~> 1.90"
   spec.add_development_dependency "simplecov", "~> 1.2"
 
-  spec.add_dependency "connection_pool", ">= 2.4", "< 4.0"
-  spec.add_dependency "deepl-rb", "~> 3.9"
-  spec.add_dependency "dry-initializer", "~> 3.2"
+  # The only two gems this one loads. Everything else -- the DeepL client, the
+  # connection pool, and whatever backs the cache and the rate limiter -- is
+  # supplied by the application and duck typed, so it stays out of the gemspec.
   spec.add_dependency "ox", "~> 2.14"
   spec.add_dependency "punkt-segmenter", "~> 0.9"
-  spec.add_dependency "ratelimit", "~> 1.1"
-  spec.add_dependency "redis", ">= 5.0", "< 7.0"
-  spec.add_dependency "redis-namespace", "~> 1.11"
 end

data/lib/deepl_diff/cache.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 class DeepLDiff::Cache
-  extend Dry::Initializer
-
-  param :from
-  param :to
+  def initialize(from, to)
+    @from = from
+    @to = to
+  end
 
   def cached_and_missing(values)
     keys = values.map { |v| key(v) }
@@ -22,6 +22,8 @@ class DeepLDiff::Cache
 
   private
 
+  attr_reader :from, :to
+
   def store_value(value, translation)
     cache_store.write(key(value), translation)
     translation

data/lib/deepl_diff/chunker.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 class DeepLDiff::Chunker
-  extend ::Dry::Initializer
-
   class Error < StandardError; end
 
-  Chunk = Struct.new(:texts, :bytesize)
+  Chunk = Struct.new(:texts, :escaped_size)
 
   MAX_CHUNK_SIZE = 1700
   COUNT_LIMIT = 300
 
-  param :values
-  option :limit, default: proc { MAX_CHUNK_SIZE }
-  option :count_limit, default: proc { COUNT_LIMIT }
+  def initialize(values, limit: MAX_CHUNK_SIZE, count_limit: COUNT_LIMIT)
+    @values = values
+    @limit = limit
+    @count_limit = count_limit
+  end
 
   def call
     chunks.map(&:texts)
@@ -35,22 +35,29 @@ class DeepLDiff::Chunker
 
   private
 
+  attr_reader :values, :limit, :count_limit
+
   def next_chunk?(tail, value)
     tail.nil? ||
-      (size(value) + tail.bytesize > limit) ||
-      tail.texts.size > count_limit
+      (escaped_size(value) + tail.escaped_size > limit) ||
+      tail.texts.size >= count_limit
   end
 
-  def size(text)
+  # What the limit is about is the size of the request that goes over the
+  # wire, so every measurement here is of the escaped form. Mixing it with
+  # String#size lets a chunk of non-ASCII text run several times over.
+  def escaped_size(text)
     CGI.escape(text).size
   end
 
   def update_chunk(chunk, value)
     chunk.texts << value
-    chunk.bytesize += value.size
+    chunk.escaped_size += escaped_size(value)
   end
 
   def validate_value_size(value)
-    raise Error, "Too long part #{value.size} > #{limit}" if value.size > limit
+    size = escaped_size(value)
+
+    raise Error, "Too long part #{size} > #{limit}" if size > limit
   end
 end

data/lib/deepl_diff/redis_cache_store.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 class DeepLDiff::RedisCacheStore
-  extend Dry::Initializer
+  ONE_WEEK = 60 * 60 * 24 * 7
 
-  param :connection_pool
-
-  option :timeout, default: proc { 60 * 60 * 24 * 7 }
-  option :namespace, default: proc { DeepLDiff::CACHE_NAMESPACE }
+  # `connection_pool` is anything answering to #with, and what it yields is
+  # anything Redis::Namespace accepts. Neither gem is a dependency of this one.
+  def initialize(connection_pool, timeout: ONE_WEEK, namespace: DeepLDiff::CACHE_NAMESPACE)
+    @connection_pool = connection_pool
+    @timeout = timeout
+    @namespace = namespace
+  end
 
   def read_multi(keys)
     redis { |redis| redis.mget(*keys) }
@@ -18,6 +21,8 @@ class DeepLDiff::RedisCacheStore
 
   private
 
+  attr_reader :connection_pool, :timeout, :namespace
+
   def redis
     connection_pool.with do |redis|
       yield Redis::Namespace.new(namespace, redis: redis)

data/lib/deepl_diff/redis_rate_limiter.rb

@@ -1,15 +1,22 @@
 # frozen_string_literal: true
 
 class DeepLDiff::RedisRateLimiter
-  extend Dry::Initializer
-
   class RateLimitExceeded < StandardError; end
 
-  param :connection_pool
-  param :threshold, default: proc { 8000 }
-  param :interval,  default: proc { 60 }
+  DEFAULT_THRESHOLD = 8000
+  DEFAULT_INTERVAL = 60
 
-  option :namespace, default: proc { DeepLDiff::CACHE_NAMESPACE }
+  # `connection_pool` is anything answering to #with, and what it yields is
+  # anything Ratelimit accepts. Neither gem is a dependency of this one.
+  def initialize(connection_pool,
+                 threshold: DEFAULT_THRESHOLD,
+                 interval: DEFAULT_INTERVAL,
+                 namespace: DeepLDiff::CACHE_NAMESPACE)
+    @connection_pool = connection_pool
+    @threshold = threshold
+    @interval = interval
+    @namespace = namespace
+  end
 
   def check(size)
     connection_pool.with do |redis|
@@ -19,4 +26,8 @@ class DeepLDiff::RedisRateLimiter
       rate_limit.add size
     end
   end
+
+  private
+
+  attr_reader :connection_pool, :threshold, :interval, :namespace
 end

data/lib/deepl_diff/request.rb

@@ -1,25 +1,32 @@
 # frozen_string_literal: true
 
 class DeepLDiff::Request
-  extend Dry::Initializer
   extend Forwardable
 
-  param :values
-  param :options
+  class Error < StandardError; end
 
   def_delegators :DeepLDiff, :api, :cache_store, :rate_limiter
   def_delegators :"DeepLDiff::Linearizer", :linearize, :restore
 
+  def initialize(values, options)
+    @values = values
+    # #from and #to consume their keys so the rest can go to the API as-is.
+    # Copy first: the caller's hash is theirs, and it is often frozen.
+    @options = options.dup
+  end
+
   def call
     validate_globals
 
-    return values if from == to || values.empty?
+    return values if same_language? || nothing_to_translate?
 
     translation
   end
 
   private
 
+  attr_reader :values, :options
+
   def from
     @from ||= options.delete(:from) || detect_language
   end
@@ -28,6 +35,19 @@ class DeepLDiff::Request
     @to ||= options.delete(:to) { nil }
   end
 
+  # A detected language arrives as a String while :to is usually a Symbol, so
+  # the two have to be compared on equal footing or the short circuit never
+  # fires and the text gets translated into its own language.
+  def same_language?
+    !to.nil? && from.to_s.casecmp?(to.to_s)
+  end
+
+  # Covers values holding no translatable text at all: "", nil, an empty
+  # collection, or a scalar the tokenizer has nothing to say about.
+  def nothing_to_translate?
+    text_tokens_texts.all?(&:empty?)
+  end
+
   def detect_language
     api.translate(text_tokens_texts.join(" ")[0..100], nil, to)
        .detected_source_language.downcase
@@ -123,7 +143,12 @@ class DeepLDiff::Request
   # Restores texts from tokens
   # [..., "<b>Horoshiy</b> Malchik", ...]
   def texts_translated
-    @texts_translated ||= tokens_translated.map do |group|
+    @texts_translated ||= tokens_translated.map.with_index do |group, index|
+      source = texts[index]
+      # Only strings are rebuilt from tokens. Anything else has no tokens to
+      # rebuild from; nil keeps collapsing to "" the way it always has.
+      next source unless source.nil? || source.is_a?(String)
+
       group.map { |value, type| type == :text ? value : fix_ascii(value) }.join
     end
   end
@@ -135,7 +160,13 @@ class DeepLDiff::Request
 
   def call_api(values)
     check_rate_limit(values)
-    [api.translate(values, from, to, options)].flatten.map(&:text)
+    translations = [api.translate(values, from, to, options)].flatten.map(&:text)
+    return translations if translations.size == values.size
+
+    # Letting a short response through means shifting nils into the results,
+    # which surfaces much later as a NoMethodError far from the cause.
+    raise Error,
+          "API returned #{translations.size} translations for #{values.size} values"
   end
 
   def cache

data/lib/deepl_diff/tokenizer.rb

@@ -149,7 +149,8 @@ class DeepLDiff::Tokenizer < Ox::Sax
 
   class << self
     def tokenize(value)
-      return [] if value.nil?
+      # Anything that is not a string has no markup and no sentences in it.
+      return [] unless value.is_a?(String)
 
       tokenizer = new(value).tap do |h|
         Ox.sax_parse(h, StringIO.new(value), HTML_OPTIONS)

data/lib/deepl_diff/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeepLDiff
-  VERSION = "1.1.1"
+  VERSION = "2.1.0"
 end

data/lib/deepl_diff.rb

@@ -7,8 +7,6 @@ require "stringio"
 
 require "ox"
 require "punkt-segmenter"
-require "dry/initializer"
-require "deepl"
 
 require "deepl_diff/version"
 require "deepl_diff/tokenizer"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deepl_diff
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Islam Gagiev
@@ -65,54 +65,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: connection_pool
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: deepl-rb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.9'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.9'
-- !ruby/object:Gem::Dependency
-  name: dry-initializer
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: ox
   requirement: !ruby/object:Gem::Requirement
@@ -141,54 +93,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: ratelimit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '7.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '7.0'
-- !ruby/object:Gem::Dependency
-  name: redis-namespace
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
 description: "\nDeepL API wrapper for Ruby which helps to translate only changes\nbetween
   revisions of long texts.\n  "
 email: