sidekiq-grouping 1.2.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Appraisals +8 -0
- data/bin/console +8 -0
- data/gemfiles/sidekiq_7.0.gemfile +7 -0
- data/lefthook.yml +7 -2
- data/lib/sidekiq/grouping/config.rb +5 -3
- data/lib/sidekiq/grouping/redis.rb +43 -29
- data/lib/sidekiq/grouping/redis_dispatcher.rb +29 -0
- data/lib/sidekiq/grouping/version.rb +1 -1
- data/lib/sidekiq/grouping.rb +1 -0
- data/sidekiq-grouping.gemspec +2 -1
- data/spec/modules/redis_spec.rb +9 -13
- data/spec/spec_helper.rb +12 -4
- metadata +20 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1626ba86eaf1cb747b25ca6770aa5fa982fbc19f4b8f55bdb49983331ffa8761
|
4
|
+
data.tar.gz: bdd1ee7ec9d60582fdf2db69edacbb5a3592f1caa77d60ea60e09f259a0dbca4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 99cfd0e831a581d1f38764f1803e81fcdd4ae5b2fd7a75fa8b19a936af601abc415031213f8419d16f3108d8ba8a980439b148e55ae2b64ebc89e674d4c2b780
|
7
|
+
data.tar.gz: 9b98e1272e011f68fc60766918f16cb80ce867eb351d05b65ac8a760dbfe05fe363b74a276794b80a6971e5b4593d2c423d35ad835ad7bbdb00d6ca911490e94
|
data/.gitignore
CHANGED
data/Appraisals
CHANGED
data/bin/console
ADDED
data/lefthook.yml
CHANGED
@@ -4,12 +4,17 @@
|
|
4
4
|
# See: github.com/evilmartians/lefthook
|
5
5
|
|
6
6
|
pre-commit:
|
7
|
+
parallel: true
|
7
8
|
commands:
|
9
|
+
appraisal:
|
10
|
+
glob: "{Appraisals,*.gemfile}"
|
11
|
+
run: echo {staged_files} > /dev/null; bundle exec appraisal install && git add gemfiles/*.gemfile
|
8
12
|
rubocop:
|
9
|
-
glob: "*.rb"
|
13
|
+
glob: "{*.rb,*.gemspec,Gemfile,Rakefile}"
|
10
14
|
run: bundle exec rubocop -A {staged_files} && git add {staged_files}
|
11
15
|
|
12
16
|
pre-push:
|
13
17
|
commands:
|
14
18
|
rspec:
|
15
|
-
|
19
|
+
glob: "*.rb"
|
20
|
+
run: echo {push_files} > /dev/null; bundle exec appraisal rspec
|
@@ -6,10 +6,12 @@ module Sidekiq
|
|
6
6
|
include ActiveSupport::Configurable
|
7
7
|
|
8
8
|
def self.options
|
9
|
-
if Sidekiq.respond_to?(:[])
|
10
|
-
Sidekiq[:grouping] ||
|
11
|
-
|
9
|
+
if Sidekiq.respond_to?(:[]) # Sidekiq 6.x
|
10
|
+
Sidekiq[:grouping] || {}
|
11
|
+
elsif Sidekiq.respond_to?(:options) # Sidekiq <= 5.x
|
12
12
|
Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {}
|
13
|
+
else # Sidekiq 7.x
|
14
|
+
Sidekiq.default_configuration[:grouping] || {}
|
13
15
|
end
|
14
16
|
end
|
15
17
|
|
@@ -1,8 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "./redis_dispatcher"
|
4
|
+
|
3
5
|
module Sidekiq
|
4
6
|
module Grouping
|
5
7
|
class Redis
|
8
|
+
include RedisDispatcher
|
9
|
+
|
6
10
|
PLUCK_SCRIPT = <<-SCRIPT
|
7
11
|
local pluck_values = redis.call('lpop', KEYS[1], ARGV[1]) or {}
|
8
12
|
if #pluck_values > 0 then
|
@@ -15,13 +19,12 @@ module Sidekiq
|
|
15
19
|
redis do |conn|
|
16
20
|
conn.multi do |pipeline|
|
17
21
|
sadd = pipeline.respond_to?(:sadd?) ? :sadd? : :sadd
|
18
|
-
pipeline
|
19
|
-
pipeline
|
22
|
+
redis_connection_call(pipeline, sadd, ns("batches"), name)
|
23
|
+
redis_connection_call(pipeline, :rpush, ns(name), msg)
|
24
|
+
|
20
25
|
if remember_unique
|
21
|
-
|
22
|
-
sadd,
|
23
|
-
unique_messages_key(name),
|
24
|
-
msg
|
26
|
+
redis_connection_call(
|
27
|
+
pipeline, sadd, unique_messages_key(name), msg
|
25
28
|
)
|
26
29
|
end
|
27
30
|
end
|
@@ -29,47 +32,62 @@ module Sidekiq
|
|
29
32
|
end
|
30
33
|
|
31
34
|
def enqueued?(name, msg)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
+
member = redis_call(:sismember, unique_messages_key(name), msg)
|
36
|
+
return member if member.is_a?(TrueClass) || member.is_a?(FalseClass)
|
37
|
+
|
38
|
+
member != 0
|
35
39
|
end
|
36
40
|
|
37
41
|
def batch_size(name)
|
38
|
-
|
42
|
+
redis_call(:llen, ns(name))
|
39
43
|
end
|
40
44
|
|
41
45
|
def batches
|
42
|
-
|
46
|
+
redis_call(:smembers, ns("batches"))
|
43
47
|
end
|
44
48
|
|
45
49
|
def pluck(name, limit)
|
46
|
-
|
47
|
-
|
48
|
-
|
50
|
+
if new_redis_client?
|
51
|
+
redis_call(
|
52
|
+
:eval,
|
53
|
+
PLUCK_SCRIPT,
|
54
|
+
2,
|
55
|
+
ns(name),
|
56
|
+
unique_messages_key(name),
|
57
|
+
limit
|
58
|
+
)
|
59
|
+
else
|
60
|
+
keys = [ns(name), unique_messages_key(name)]
|
61
|
+
args = [limit]
|
62
|
+
redis_call(:eval, PLUCK_SCRIPT, keys, args)
|
63
|
+
end
|
49
64
|
end
|
50
65
|
|
51
66
|
def get_last_execution_time(name)
|
52
|
-
|
67
|
+
redis_call(:get, ns("last_execution_time:#{name}"))
|
53
68
|
end
|
54
69
|
|
55
70
|
def set_last_execution_time(name, time)
|
56
|
-
|
57
|
-
|
58
|
-
|
71
|
+
redis_call(
|
72
|
+
:set, ns("last_execution_time:#{name}"), time.to_json
|
73
|
+
)
|
59
74
|
end
|
60
75
|
|
61
76
|
def lock(name)
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
77
|
+
redis_call(
|
78
|
+
:set,
|
79
|
+
ns("lock:#{name}"),
|
80
|
+
"true",
|
81
|
+
nx: true,
|
82
|
+
ex: Sidekiq::Grouping::Config.lock_ttl
|
83
|
+
)
|
66
84
|
end
|
67
85
|
|
68
86
|
def delete(name)
|
69
87
|
redis do |conn|
|
70
|
-
conn
|
71
|
-
conn
|
72
|
-
conn
|
88
|
+
redis_connection_call(conn, :del, ns("last_execution_time:#{name}"))
|
89
|
+
redis_connection_call(conn, :del, ns(name))
|
90
|
+
redis_connection_call(conn, :srem, ns("batches"), name)
|
73
91
|
end
|
74
92
|
end
|
75
93
|
|
@@ -82,10 +100,6 @@ module Sidekiq
|
|
82
100
|
def ns(key = nil)
|
83
101
|
"batching:#{key}"
|
84
102
|
end
|
85
|
-
|
86
|
-
def redis(&block)
|
87
|
-
Sidekiq.redis(&block)
|
88
|
-
end
|
89
103
|
end
|
90
104
|
end
|
91
105
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Grouping
|
5
|
+
module RedisDispatcher
|
6
|
+
def redis_call(command, *args, **kwargs)
|
7
|
+
redis do |connection|
|
8
|
+
redis_connection_call(connection, command, *args, **kwargs)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def redis_connection_call(connection, command, *args, **kwargs)
|
13
|
+
if new_redis_client? # redis-client
|
14
|
+
connection.call(command.to_s.upcase, *args, **kwargs)
|
15
|
+
else # redis
|
16
|
+
connection.public_send(command, *args, **kwargs)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def new_redis_client?
|
21
|
+
Sidekiq::VERSION[0].to_i >= 7
|
22
|
+
end
|
23
|
+
|
24
|
+
def redis(&block)
|
25
|
+
Sidekiq.redis(&block)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/sidekiq/grouping.rb
CHANGED
data/sidekiq-grouping.gemspec
CHANGED
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
|
24
24
|
spec.add_development_dependency "appraisal"
|
25
25
|
spec.add_development_dependency "bundler", "> 1.5"
|
26
|
+
spec.add_development_dependency "pry"
|
26
27
|
spec.add_development_dependency "rake"
|
27
28
|
spec.add_development_dependency "rspec"
|
28
29
|
spec.add_development_dependency "rspec-sidekiq"
|
@@ -33,6 +34,6 @@ Gem::Specification.new do |spec|
|
|
33
34
|
|
34
35
|
spec.add_dependency "activesupport"
|
35
36
|
spec.add_dependency "concurrent-ruby"
|
36
|
-
spec.add_dependency "sidekiq", ">= 3.4.2"
|
37
|
+
spec.add_dependency "sidekiq", ">= 3.4.2"
|
37
38
|
spec.metadata["rubygems_mfa_required"] = "true"
|
38
39
|
end
|
data/spec/modules/redis_spec.rb
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
require "spec_helper"
|
4
4
|
|
5
5
|
describe Sidekiq::Grouping::Redis do
|
6
|
+
include Sidekiq::Grouping::RedisDispatcher
|
7
|
+
|
6
8
|
subject(:redis_service) { described_class.new }
|
7
9
|
|
8
10
|
let(:queue_name) { "my_queue" }
|
@@ -12,14 +14,14 @@ describe Sidekiq::Grouping::Redis do
|
|
12
14
|
describe "#push_msg" do
|
13
15
|
it "adds message to queue", :aggregate_failures do
|
14
16
|
redis_service.push_msg(queue_name, "My message")
|
15
|
-
expect(
|
16
|
-
expect(
|
17
|
-
expect(
|
17
|
+
expect(redis_call(:llen, key)).to eq 1
|
18
|
+
expect(redis_call(:lrange, key, 0, 1)).to eq ["My message"]
|
19
|
+
expect(redis_call(:smembers, unique_key)).to eq []
|
18
20
|
end
|
19
21
|
|
20
22
|
it "remembers unique message if specified" do
|
21
23
|
redis_service.push_msg(queue_name, "My message", remember_unique: true)
|
22
|
-
expect(
|
24
|
+
expect(redis_call(:smembers, unique_key)).to eq ["My message"]
|
23
25
|
end
|
24
26
|
end
|
25
27
|
|
@@ -28,21 +30,15 @@ describe Sidekiq::Grouping::Redis do
|
|
28
30
|
redis_service.push_msg(queue_name, "Message 1")
|
29
31
|
redis_service.push_msg(queue_name, "Message 2")
|
30
32
|
redis_service.pluck(queue_name, 2)
|
31
|
-
expect(
|
33
|
+
expect(redis_call(:llen, key)).to eq 0
|
32
34
|
end
|
33
35
|
|
34
36
|
it "forgets unique messages", :aggregate_failures do
|
35
37
|
redis_service.push_msg(queue_name, "Message 1", remember_unique: true)
|
36
38
|
redis_service.push_msg(queue_name, "Message 2", remember_unique: true)
|
37
|
-
expect(
|
39
|
+
expect(redis_call(:scard, unique_key)).to eq 2
|
38
40
|
redis_service.pluck(queue_name, 2)
|
39
|
-
expect(
|
41
|
+
expect(redis_call(:smembers, unique_key)).to eq []
|
40
42
|
end
|
41
43
|
end
|
42
|
-
|
43
|
-
private
|
44
|
-
|
45
|
-
def redis(&block)
|
46
|
-
Sidekiq.redis(&block)
|
47
|
-
end
|
48
44
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -9,6 +9,7 @@ require "simplecov"
|
|
9
9
|
require "sidekiq"
|
10
10
|
require "rspec-sidekiq"
|
11
11
|
require "support/test_workers"
|
12
|
+
require "pry"
|
12
13
|
|
13
14
|
SimpleCov.start do
|
14
15
|
add_filter "spec"
|
@@ -17,8 +18,10 @@ end
|
|
17
18
|
require "sidekiq/grouping"
|
18
19
|
|
19
20
|
Sidekiq::Grouping.logger = nil
|
20
|
-
Sidekiq.
|
21
|
-
|
21
|
+
Sidekiq.configure_client do |config|
|
22
|
+
config.redis = { db: 1 }
|
23
|
+
config.logger = nil
|
24
|
+
end
|
22
25
|
|
23
26
|
RSpec::Sidekiq.configure do |config|
|
24
27
|
config.clear_all_enqueued_jobs = true
|
@@ -32,8 +35,13 @@ RSpec.configure do |config|
|
|
32
35
|
|
33
36
|
config.before do
|
34
37
|
Sidekiq.redis do |conn|
|
35
|
-
|
36
|
-
|
38
|
+
if Sidekiq::VERSION[0].to_i >= 7
|
39
|
+
keys = conn.call("KEYS", "*batching*")
|
40
|
+
keys.each { |key| conn.call("DEL", key) }
|
41
|
+
else
|
42
|
+
keys = conn.keys "*batching*"
|
43
|
+
keys.each { |key| conn.del key }
|
44
|
+
end
|
37
45
|
end
|
38
46
|
end
|
39
47
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-grouping
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Sokolov
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.5'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: pry
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: rake
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -171,9 +185,6 @@ dependencies:
|
|
171
185
|
- - ">="
|
172
186
|
- !ruby/object:Gem::Version
|
173
187
|
version: 3.4.2
|
174
|
-
- - "<"
|
175
|
-
- !ruby/object:Gem::Version
|
176
|
-
version: '7'
|
177
188
|
type: :runtime
|
178
189
|
prerelease: false
|
179
190
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -181,13 +192,11 @@ dependencies:
|
|
181
192
|
- - ">="
|
182
193
|
- !ruby/object:Gem::Version
|
183
194
|
version: 3.4.2
|
184
|
-
- - "<"
|
185
|
-
- !ruby/object:Gem::Version
|
186
|
-
version: '7'
|
187
195
|
description:
|
188
196
|
email:
|
189
197
|
- gzigzigzeo@gmail.com
|
190
|
-
executables:
|
198
|
+
executables:
|
199
|
+
- console
|
191
200
|
extensions: []
|
192
201
|
extra_rdoc_files: []
|
193
202
|
files:
|
@@ -200,12 +209,14 @@ files:
|
|
200
209
|
- LICENSE.txt
|
201
210
|
- README.md
|
202
211
|
- Rakefile
|
212
|
+
- bin/console
|
203
213
|
- gemfiles/sidekiq_4.0.gemfile
|
204
214
|
- gemfiles/sidekiq_4.1.gemfile
|
205
215
|
- gemfiles/sidekiq_4.2.gemfile
|
206
216
|
- gemfiles/sidekiq_5.0.gemfile
|
207
217
|
- gemfiles/sidekiq_6.0.gemfile
|
208
218
|
- gemfiles/sidekiq_6.5.gemfile
|
219
|
+
- gemfiles/sidekiq_7.0.gemfile
|
209
220
|
- gemfiles/sidekiq_master.gemfile
|
210
221
|
- lefthook.yml
|
211
222
|
- lib/sidekiq/grouping.rb
|
@@ -215,6 +226,7 @@ files:
|
|
215
226
|
- lib/sidekiq/grouping/flusher_observer.rb
|
216
227
|
- lib/sidekiq/grouping/middleware.rb
|
217
228
|
- lib/sidekiq/grouping/redis.rb
|
229
|
+
- lib/sidekiq/grouping/redis_dispatcher.rb
|
218
230
|
- lib/sidekiq/grouping/version.rb
|
219
231
|
- lib/sidekiq/grouping/views/index.erb
|
220
232
|
- lib/sidekiq/grouping/web.rb
|