deepl_diff 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/.github/workflows/release.yml +1 -1
- data/.gitignore +1 -3
- data/.rubocop.yml +4 -1
- data/README.md +17 -4
- data/Rakefile +9 -3
- data/deepl_diff.gemspec +4 -7
- data/lib/deepl_diff/cache.rb +6 -4
- data/lib/deepl_diff/chunker.rb +7 -5
- data/lib/deepl_diff/redis_cache_store.rb +10 -5
- data/lib/deepl_diff/redis_rate_limiter.rb +17 -6
- data/lib/deepl_diff/request.rb +7 -4
- data/lib/deepl_diff/version.rb +1 -1
- data/lib/deepl_diff.rb +0 -2
- metadata +7 -104
- data/.rspec +0 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ca22132db1cfc83ab8e56a9a17bd635ca3648e9d7fde32a2df13dce21991f946
|
|
4
|
+
data.tar.gz: '018f598439a77db2e229d2c95f7b8ff0c14c0f17ba7130e19cbdf3f897b716a6'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0144f0c5d615c25036d0f2f2b41238667e6ab045442714e45235c8192f49c70fdeeb92753c6e7ec5f0e1189de3064191bdbb2014108a5715942ee1455f6ae76f
|
|
7
|
+
data.tar.gz: 71a0a99b3f5869bec1f2383f8d212c6ead9bd63f9e5eb5a9f8ad98ed2aa93f0701d5775b9de3764c21aad7ac96f11afe2a506e674056f9a97a80ed5d357c65b6
|
data/.github/workflows/ci.yml
CHANGED
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -13,7 +13,7 @@ Style/ClassAndModuleChildren:
|
|
|
13
13
|
|
|
14
14
|
Metrics/BlockLength:
|
|
15
15
|
Exclude:
|
|
16
|
-
-
|
|
16
|
+
- test/**/*
|
|
17
17
|
|
|
18
18
|
Layout/LineLength:
|
|
19
19
|
Max: 120
|
|
@@ -24,3 +24,6 @@ Gemspec/DevelopmentDependencies:
|
|
|
24
24
|
Metrics/ClassLength:
|
|
25
25
|
Exclude:
|
|
26
26
|
- lib/deepl_diff/tokenizer.rb
|
|
27
|
+
- lib/deepl_diff/request.rb
|
|
28
|
+
# Test classes are mostly tables of cases.
|
|
29
|
+
- test/**/*
|
data/README.md
CHANGED
|
@@ -17,6 +17,16 @@ If your user changes a single word within the long description, you will be char
|
|
|
17
17
|
|
|
18
18
|
Much better approach is to try to translate every repeated structural element (sentence) in your texts array just once to save money. This gem helps to make it done.
|
|
19
19
|
|
|
20
|
+
## Dependencies
|
|
21
|
+
|
|
22
|
+
This gem loads two: [`ox`](https://github.com/ohler55/ox) to walk the HTML, and
|
|
23
|
+
[`punkt-segmenter`](https://github.com/lfcipriani/punkt-segmenter) to split text
|
|
24
|
+
into sentences.
|
|
25
|
+
|
|
26
|
+
Everything else is duck typed and supplied by you: `DeepLDiff.api` is anything
|
|
27
|
+
answering to `#translate`, and both `RedisCacheStore` and `RedisRateLimiter`
|
|
28
|
+
take anything answering to `#with`. Bring your own client, pool and store.
|
|
29
|
+
|
|
20
30
|
## Installation
|
|
21
31
|
|
|
22
32
|
Add this line to your application's Gemfile:
|
|
@@ -38,11 +48,14 @@ Or install it yourself as:
|
|
|
38
48
|
```ruby
|
|
39
49
|
require "deepl_diff"
|
|
40
50
|
|
|
41
|
-
#
|
|
51
|
+
# None of these are dependencies of this gem. It loads only `ox` and
|
|
52
|
+
# `punkt-segmenter`; the API client, the connection pool, and whatever backs
|
|
53
|
+
# the cache and the rate limiter are yours to choose and to require.
|
|
54
|
+
require "deepl"
|
|
42
55
|
require "redis"
|
|
43
56
|
require "connection_pool"
|
|
44
57
|
require "redis-namespace"
|
|
45
|
-
require "ratelimit" # Optional, if you
|
|
58
|
+
require "ratelimit" # Optional, only if you use the rate limiter
|
|
46
59
|
|
|
47
60
|
# Setup https://github.com/wikiti/deepl-rb
|
|
48
61
|
DeepL.configure do |config|
|
|
@@ -108,10 +121,10 @@ DeepL API has a limitation: query can not be longer than approximately 128 KB. I
|
|
|
108
121
|
|
|
109
122
|
## Development
|
|
110
123
|
|
|
111
|
-
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake
|
|
124
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
112
125
|
|
|
113
126
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
114
127
|
|
|
115
128
|
## Contributing
|
|
116
129
|
|
|
117
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
|
130
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/Halvanhelv/deepl_diff.
|
data/Rakefile
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "bundler/gem_tasks"
|
|
4
|
-
require "
|
|
4
|
+
require "rake/testtask"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
|
7
|
+
t.libs << "test"
|
|
8
|
+
t.libs << "lib"
|
|
9
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
|
10
|
+
# The dependencies are noisy under -w; their warnings drown out our own.
|
|
11
|
+
t.warning = false
|
|
12
|
+
end
|
|
7
13
|
|
|
8
|
-
task default: :
|
|
14
|
+
task default: :test
|
data/deepl_diff.gemspec
CHANGED
|
@@ -40,17 +40,14 @@ between revisions of long texts.
|
|
|
40
40
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
41
41
|
spec.require_paths = ["lib"]
|
|
42
42
|
|
|
43
|
+
spec.add_development_dependency "minitest", "~> 6.0"
|
|
43
44
|
spec.add_development_dependency "rake", "~> 13.0"
|
|
44
|
-
spec.add_development_dependency "rspec", "~> 3.13"
|
|
45
45
|
spec.add_development_dependency "rubocop", "~> 1.90"
|
|
46
46
|
spec.add_development_dependency "simplecov", "~> 1.2"
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
# The only two gems this one loads. Everything else -- the DeepL client, the
|
|
49
|
+
# connection pool, and whatever backs the cache and the rate limiter -- is
|
|
50
|
+
# supplied by the application and duck typed, so it stays out of the gemspec.
|
|
51
51
|
spec.add_dependency "ox", "~> 2.14"
|
|
52
52
|
spec.add_dependency "punkt-segmenter", "~> 0.9"
|
|
53
|
-
spec.add_dependency "ratelimit", "~> 1.1"
|
|
54
|
-
spec.add_dependency "redis", ">= 5.0", "< 7.0"
|
|
55
|
-
spec.add_dependency "redis-namespace", "~> 1.11"
|
|
56
53
|
end
|
data/lib/deepl_diff/cache.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class DeepLDiff::Cache
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
def initialize(from, to)
|
|
5
|
+
@from = from
|
|
6
|
+
@to = to
|
|
7
|
+
end
|
|
8
8
|
|
|
9
9
|
def cached_and_missing(values)
|
|
10
10
|
keys = values.map { |v| key(v) }
|
|
@@ -22,6 +22,8 @@ class DeepLDiff::Cache
|
|
|
22
22
|
|
|
23
23
|
private
|
|
24
24
|
|
|
25
|
+
attr_reader :from, :to
|
|
26
|
+
|
|
25
27
|
def store_value(value, translation)
|
|
26
28
|
cache_store.write(key(value), translation)
|
|
27
29
|
translation
|
data/lib/deepl_diff/chunker.rb
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class DeepLDiff::Chunker
|
|
4
|
-
extend ::Dry::Initializer
|
|
5
|
-
|
|
6
4
|
class Error < StandardError; end
|
|
7
5
|
|
|
8
6
|
Chunk = Struct.new(:texts, :bytesize)
|
|
@@ -10,9 +8,11 @@ class DeepLDiff::Chunker
|
|
|
10
8
|
MAX_CHUNK_SIZE = 1700
|
|
11
9
|
COUNT_LIMIT = 300
|
|
12
10
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
def initialize(values, limit: MAX_CHUNK_SIZE, count_limit: COUNT_LIMIT)
|
|
12
|
+
@values = values
|
|
13
|
+
@limit = limit
|
|
14
|
+
@count_limit = count_limit
|
|
15
|
+
end
|
|
16
16
|
|
|
17
17
|
def call
|
|
18
18
|
chunks.map(&:texts)
|
|
@@ -35,6 +35,8 @@ class DeepLDiff::Chunker
|
|
|
35
35
|
|
|
36
36
|
private
|
|
37
37
|
|
|
38
|
+
attr_reader :values, :limit, :count_limit
|
|
39
|
+
|
|
38
40
|
def next_chunk?(tail, value)
|
|
39
41
|
tail.nil? ||
|
|
40
42
|
(size(value) + tail.bytesize > limit) ||
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class DeepLDiff::RedisCacheStore
|
|
4
|
-
|
|
4
|
+
ONE_WEEK = 60 * 60 * 24 * 7
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
# `connection_pool` is anything answering to #with, and what it yields is
|
|
7
|
+
# anything Redis::Namespace accepts. Neither gem is a dependency of this one.
|
|
8
|
+
def initialize(connection_pool, timeout: ONE_WEEK, namespace: DeepLDiff::CACHE_NAMESPACE)
|
|
9
|
+
@connection_pool = connection_pool
|
|
10
|
+
@timeout = timeout
|
|
11
|
+
@namespace = namespace
|
|
12
|
+
end
|
|
10
13
|
|
|
11
14
|
def read_multi(keys)
|
|
12
15
|
redis { |redis| redis.mget(*keys) }
|
|
@@ -18,6 +21,8 @@ class DeepLDiff::RedisCacheStore
|
|
|
18
21
|
|
|
19
22
|
private
|
|
20
23
|
|
|
24
|
+
attr_reader :connection_pool, :timeout, :namespace
|
|
25
|
+
|
|
21
26
|
def redis
|
|
22
27
|
connection_pool.with do |redis|
|
|
23
28
|
yield Redis::Namespace.new(namespace, redis: redis)
|
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class DeepLDiff::RedisRateLimiter
|
|
4
|
-
extend Dry::Initializer
|
|
5
|
-
|
|
6
4
|
class RateLimitExceeded < StandardError; end
|
|
7
5
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
param :interval, default: proc { 60 }
|
|
6
|
+
DEFAULT_THRESHOLD = 8000
|
|
7
|
+
DEFAULT_INTERVAL = 60
|
|
11
8
|
|
|
12
|
-
|
|
9
|
+
# `connection_pool` is anything answering to #with, and what it yields is
|
|
10
|
+
# anything Ratelimit accepts. Neither gem is a dependency of this one.
|
|
11
|
+
def initialize(connection_pool,
|
|
12
|
+
threshold: DEFAULT_THRESHOLD,
|
|
13
|
+
interval: DEFAULT_INTERVAL,
|
|
14
|
+
namespace: DeepLDiff::CACHE_NAMESPACE)
|
|
15
|
+
@connection_pool = connection_pool
|
|
16
|
+
@threshold = threshold
|
|
17
|
+
@interval = interval
|
|
18
|
+
@namespace = namespace
|
|
19
|
+
end
|
|
13
20
|
|
|
14
21
|
def check(size)
|
|
15
22
|
connection_pool.with do |redis|
|
|
@@ -19,4 +26,8 @@ class DeepLDiff::RedisRateLimiter
|
|
|
19
26
|
rate_limit.add size
|
|
20
27
|
end
|
|
21
28
|
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
attr_reader :connection_pool, :threshold, :interval, :namespace
|
|
22
33
|
end
|
data/lib/deepl_diff/request.rb
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class DeepLDiff::Request
|
|
4
|
-
extend Dry::Initializer
|
|
5
4
|
extend Forwardable
|
|
6
5
|
|
|
7
|
-
param :values
|
|
8
|
-
param :options
|
|
9
|
-
|
|
10
6
|
def_delegators :DeepLDiff, :api, :cache_store, :rate_limiter
|
|
11
7
|
def_delegators :"DeepLDiff::Linearizer", :linearize, :restore
|
|
12
8
|
|
|
9
|
+
def initialize(values, options)
|
|
10
|
+
@values = values
|
|
11
|
+
@options = options
|
|
12
|
+
end
|
|
13
|
+
|
|
13
14
|
def call
|
|
14
15
|
validate_globals
|
|
15
16
|
|
|
@@ -20,6 +21,8 @@ class DeepLDiff::Request
|
|
|
20
21
|
|
|
21
22
|
private
|
|
22
23
|
|
|
24
|
+
attr_reader :values, :options
|
|
25
|
+
|
|
23
26
|
def from
|
|
24
27
|
@from ||= options.delete(:from) || detect_language
|
|
25
28
|
end
|
data/lib/deepl_diff/version.rb
CHANGED
data/lib/deepl_diff.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deepl_diff
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Islam Gagiev
|
|
@@ -10,33 +10,33 @@ cert_chain: []
|
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
|
-
name:
|
|
13
|
+
name: minitest
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '
|
|
18
|
+
version: '6.0'
|
|
19
19
|
type: :development
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - "~>"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '
|
|
25
|
+
version: '6.0'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: rake
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - "~>"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
32
|
+
version: '13.0'
|
|
33
33
|
type: :development
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
39
|
+
version: '13.0'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
41
|
name: rubocop
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -65,54 +65,6 @@ dependencies:
|
|
|
65
65
|
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '1.2'
|
|
68
|
-
- !ruby/object:Gem::Dependency
|
|
69
|
-
name: connection_pool
|
|
70
|
-
requirement: !ruby/object:Gem::Requirement
|
|
71
|
-
requirements:
|
|
72
|
-
- - ">="
|
|
73
|
-
- !ruby/object:Gem::Version
|
|
74
|
-
version: '2.4'
|
|
75
|
-
- - "<"
|
|
76
|
-
- !ruby/object:Gem::Version
|
|
77
|
-
version: '4.0'
|
|
78
|
-
type: :runtime
|
|
79
|
-
prerelease: false
|
|
80
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
81
|
-
requirements:
|
|
82
|
-
- - ">="
|
|
83
|
-
- !ruby/object:Gem::Version
|
|
84
|
-
version: '2.4'
|
|
85
|
-
- - "<"
|
|
86
|
-
- !ruby/object:Gem::Version
|
|
87
|
-
version: '4.0'
|
|
88
|
-
- !ruby/object:Gem::Dependency
|
|
89
|
-
name: deepl-rb
|
|
90
|
-
requirement: !ruby/object:Gem::Requirement
|
|
91
|
-
requirements:
|
|
92
|
-
- - "~>"
|
|
93
|
-
- !ruby/object:Gem::Version
|
|
94
|
-
version: '3.9'
|
|
95
|
-
type: :runtime
|
|
96
|
-
prerelease: false
|
|
97
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
98
|
-
requirements:
|
|
99
|
-
- - "~>"
|
|
100
|
-
- !ruby/object:Gem::Version
|
|
101
|
-
version: '3.9'
|
|
102
|
-
- !ruby/object:Gem::Dependency
|
|
103
|
-
name: dry-initializer
|
|
104
|
-
requirement: !ruby/object:Gem::Requirement
|
|
105
|
-
requirements:
|
|
106
|
-
- - "~>"
|
|
107
|
-
- !ruby/object:Gem::Version
|
|
108
|
-
version: '3.2'
|
|
109
|
-
type: :runtime
|
|
110
|
-
prerelease: false
|
|
111
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
112
|
-
requirements:
|
|
113
|
-
- - "~>"
|
|
114
|
-
- !ruby/object:Gem::Version
|
|
115
|
-
version: '3.2'
|
|
116
68
|
- !ruby/object:Gem::Dependency
|
|
117
69
|
name: ox
|
|
118
70
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -141,54 +93,6 @@ dependencies:
|
|
|
141
93
|
- - "~>"
|
|
142
94
|
- !ruby/object:Gem::Version
|
|
143
95
|
version: '0.9'
|
|
144
|
-
- !ruby/object:Gem::Dependency
|
|
145
|
-
name: ratelimit
|
|
146
|
-
requirement: !ruby/object:Gem::Requirement
|
|
147
|
-
requirements:
|
|
148
|
-
- - "~>"
|
|
149
|
-
- !ruby/object:Gem::Version
|
|
150
|
-
version: '1.1'
|
|
151
|
-
type: :runtime
|
|
152
|
-
prerelease: false
|
|
153
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
154
|
-
requirements:
|
|
155
|
-
- - "~>"
|
|
156
|
-
- !ruby/object:Gem::Version
|
|
157
|
-
version: '1.1'
|
|
158
|
-
- !ruby/object:Gem::Dependency
|
|
159
|
-
name: redis
|
|
160
|
-
requirement: !ruby/object:Gem::Requirement
|
|
161
|
-
requirements:
|
|
162
|
-
- - ">="
|
|
163
|
-
- !ruby/object:Gem::Version
|
|
164
|
-
version: '5.0'
|
|
165
|
-
- - "<"
|
|
166
|
-
- !ruby/object:Gem::Version
|
|
167
|
-
version: '7.0'
|
|
168
|
-
type: :runtime
|
|
169
|
-
prerelease: false
|
|
170
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
171
|
-
requirements:
|
|
172
|
-
- - ">="
|
|
173
|
-
- !ruby/object:Gem::Version
|
|
174
|
-
version: '5.0'
|
|
175
|
-
- - "<"
|
|
176
|
-
- !ruby/object:Gem::Version
|
|
177
|
-
version: '7.0'
|
|
178
|
-
- !ruby/object:Gem::Dependency
|
|
179
|
-
name: redis-namespace
|
|
180
|
-
requirement: !ruby/object:Gem::Requirement
|
|
181
|
-
requirements:
|
|
182
|
-
- - "~>"
|
|
183
|
-
- !ruby/object:Gem::Version
|
|
184
|
-
version: '1.11'
|
|
185
|
-
type: :runtime
|
|
186
|
-
prerelease: false
|
|
187
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
188
|
-
requirements:
|
|
189
|
-
- - "~>"
|
|
190
|
-
- !ruby/object:Gem::Version
|
|
191
|
-
version: '1.11'
|
|
192
96
|
description: "\nDeepL API wrapper for Ruby which helps to translate only changes\nbetween
|
|
193
97
|
revisions of long texts.\n "
|
|
194
98
|
email:
|
|
@@ -200,7 +104,6 @@ files:
|
|
|
200
104
|
- ".github/workflows/ci.yml"
|
|
201
105
|
- ".github/workflows/release.yml"
|
|
202
106
|
- ".gitignore"
|
|
203
|
-
- ".rspec"
|
|
204
107
|
- ".rubocop.yml"
|
|
205
108
|
- ".ruby-version"
|
|
206
109
|
- Gemfile
|
data/.rspec
DELETED