sidekiq-grouping 1.0.8 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +11 -0
- data/Appraisals +19 -0
- data/README.md +67 -5
- data/gemfiles/sidekiq_4.0.gemfile +7 -0
- data/gemfiles/sidekiq_4.1.gemfile +7 -0
- data/gemfiles/sidekiq_4.2.gemfile +7 -0
- data/gemfiles/sidekiq_5.0.gemfile +7 -0
- data/gemfiles/sidekiq_master.gemfile +7 -0
- data/lib/sidekiq/grouping/config.rb +20 -6
- data/lib/sidekiq/grouping/flusher.rb +25 -3
- data/lib/sidekiq/grouping/redis.rb +5 -5
- data/lib/sidekiq/grouping/version.rb +1 -1
- data/lib/sidekiq/grouping/web.rb +3 -4
- data/lib/sidekiq/grouping.rb +5 -0
- data/sidekiq-grouping.gemspec +2 -1
- data/spec/modules/batch_spec.rb +5 -3
- metadata +29 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4dbe0fe595dfbe28a39a687925d5078d7c1df9d7793df783cf4ed448910136f9
|
4
|
+
data.tar.gz: a2f48474c1ec7ec9a632999f17cf6159d15490dfa6d06ff4eb5b85221fec776e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fb1a5cc739670f30b05cd477c3797eca72d0469307ebf4e8ea13841d49ed050b2a0814f11e29b81ce5ce7d039e99f6661110b528d8ccf22cd609654fd445b4b0
|
7
|
+
data.tar.gz: 31f85b457287ae7e1dee1c52e008099f9806cd520132c7bad048a42d4c67ee0550cb51610d95059b3135c7a39b2865058f2d2092623864b71c23010bebc03017
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -1,7 +1,18 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
3
|
- 2.2
|
4
|
+
- 2.3.1
|
5
|
+
- 2.4
|
6
|
+
- 2.5
|
7
|
+
|
4
8
|
cache: bundler
|
5
9
|
|
6
10
|
services:
|
7
11
|
- redis-server
|
12
|
+
|
13
|
+
gemfile:
|
14
|
+
- gemfiles/sidekiq_4.0.gemfile
|
15
|
+
- gemfiles/sidekiq_4.1.gemfile
|
16
|
+
- gemfiles/sidekiq_4.2.gemfile
|
17
|
+
- gemfiles/sidekiq_5.0.gemfile
|
18
|
+
- gemfiles/sidekiq_master.gemfile
|
data/Appraisals
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
appraise 'sidekiq-4.0' do
|
2
|
+
gem 'sidekiq', '~> 4.0.0'
|
3
|
+
end
|
4
|
+
|
5
|
+
appraise 'sidekiq-4.1' do
|
6
|
+
gem 'sidekiq', '~> 4.1.0'
|
7
|
+
end
|
8
|
+
|
9
|
+
appraise 'sidekiq-4.2' do
|
10
|
+
gem 'sidekiq', '~> 4.2.0'
|
11
|
+
end
|
12
|
+
|
13
|
+
appraise 'sidekiq-5.0' do
|
14
|
+
gem 'sidekiq', '~> 5.0.0'
|
15
|
+
end
|
16
|
+
|
17
|
+
appraise 'sidekiq-master' do
|
18
|
+
gem 'sidekiq', github: 'mperham/sidekiq'
|
19
|
+
end
|
data/README.md
CHANGED
@@ -111,6 +111,8 @@ This jobs will be grouped into the single job with the single argument:
|
|
111
111
|
# => [[5]]
|
112
112
|
```
|
113
113
|
|
114
|
+
- `tests_env` is used to silence some logging in test environments (see below). Default: true if `Rails.env.test?`, false otherwise.
|
115
|
+
|
114
116
|
## Web UI
|
115
117
|
|
116
118
|
![Web UI](web.png)
|
@@ -123,15 +125,75 @@ require "sidekiq/grouping/web"
|
|
123
125
|
|
124
126
|
## Configuration
|
125
127
|
|
128
|
+
Specify grouping configuration inside of sidekiq.yml:
|
129
|
+
|
130
|
+
```yml
|
131
|
+
grouping:
|
132
|
+
:poll_interval: 5 # Amount of time between polling batches
|
133
|
+
:max_batch_size: 5000 # Maximum batch size allowed
|
134
|
+
:lock_ttl: 1 # Batch queue flush lock timeout job enqueues
|
135
|
+
```
|
136
|
+
|
137
|
+
Or set it in your code:
|
138
|
+
|
126
139
|
```ruby
|
127
|
-
Sidekiq::Grouping::Config.poll_interval = 5
|
128
|
-
Sidekiq::Grouping::Config.max_batch_size = 5000
|
129
|
-
Sidekiq::Grouping::Config.lock_ttl = 1
|
140
|
+
Sidekiq::Grouping::Config.poll_interval = 5
|
141
|
+
Sidekiq::Grouping::Config.max_batch_size = 5000
|
142
|
+
Sidekiq::Grouping::Config.lock_ttl = 1
|
130
143
|
```
|
131
144
|
|
132
|
-
|
145
|
+
Note that you should set poll_interval option inside of sidekiq.yml to take effect. Setting this param in your ruby code won't change actual polling frequency.
|
146
|
+
|
147
|
+
## Testing with Sidekiq::Testing.fake!
|
148
|
+
|
149
|
+
Sidekiq::Grouping uses internal queues for grouping tasks. If you need to force flush internal queues into normal Sidekiq queues, use `Sidekiq::Grouping.force_flush_for_test!`.
|
150
|
+
|
151
|
+
See example:
|
152
|
+
|
153
|
+
```ruby
|
154
|
+
# worker
|
155
|
+
class GroupedWorker
|
156
|
+
|
157
|
+
include Sidekiq::Worker
|
158
|
+
sidekiq_options(
|
159
|
+
queue: :custom_queue,
|
160
|
+
retry: 5,
|
161
|
+
batch_flush_size: 9,
|
162
|
+
batch_flush_interval: 10,
|
163
|
+
batch_size: 3,
|
164
|
+
batch_unique: true
|
165
|
+
)
|
133
166
|
|
134
|
-
|
167
|
+
def perform(grouped_arguments)
|
168
|
+
# ... important payload
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
|
173
|
+
# test itself
|
174
|
+
RSpec.describe GroupedWorker, type: :worker do
|
175
|
+
|
176
|
+
describe '#perform' do
|
177
|
+
it 'calls perform with array of arguments' do
|
178
|
+
Sidekiq::Testing.fake! do
|
179
|
+
described_class.perform_async(1)
|
180
|
+
described_class.perform_async(1)
|
181
|
+
described_class.perform_async(2)
|
182
|
+
described_class.perform_async(2)
|
183
|
+
|
184
|
+
# All 4 above asks will be put to :custom_queue despite of :batch_flush_size is set to 9.
|
185
|
+
Sidekiq::Grouping.force_flush_for_test!
|
186
|
+
|
187
|
+
last_job = described_class.jobs.last
|
188
|
+
expect(last_job['args']).to eq([[[1], [2]]])
|
189
|
+
expect(last_job['queue']).to eq('custom_queue')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
|
196
|
+
```
|
135
197
|
|
136
198
|
## Installation
|
137
199
|
|
@@ -1,15 +1,29 @@
|
|
1
1
|
module Sidekiq::Grouping::Config
|
2
2
|
include ActiveSupport::Configurable
|
3
3
|
|
4
|
+
def self.options
|
5
|
+
Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {} # sidekiq 5.x use symbol in keys
|
6
|
+
end
|
7
|
+
|
4
8
|
# Queue size overflow check polling interval
|
5
|
-
config_accessor :poll_interval
|
6
|
-
|
9
|
+
config_accessor :poll_interval do
|
10
|
+
options[:poll_interval] || 3
|
11
|
+
end
|
7
12
|
|
8
13
|
# Maximum batch size
|
9
|
-
config_accessor :max_batch_size
|
10
|
-
|
14
|
+
config_accessor :max_batch_size do
|
15
|
+
options[:max_batch_size] || 1000
|
16
|
+
end
|
11
17
|
|
12
18
|
# Batch queue flush lock timeout
|
13
|
-
config_accessor :lock_ttl
|
14
|
-
|
19
|
+
config_accessor :lock_ttl do
|
20
|
+
options[:lock_ttl] || 1
|
21
|
+
end
|
22
|
+
|
23
|
+
# Option to override how Sidekiq::Grouping know about tests env
|
24
|
+
config_accessor :tests_env do
|
25
|
+
options[:tests_env] || (
|
26
|
+
defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
|
27
|
+
)
|
28
|
+
end
|
15
29
|
end
|
@@ -3,18 +3,40 @@ class Sidekiq::Grouping::Flusher
|
|
3
3
|
batches = Sidekiq::Grouping::Batch.all.map do |batch|
|
4
4
|
batch if batch.could_flush?
|
5
5
|
end
|
6
|
-
batches
|
7
|
-
|
6
|
+
flush_batches(batches)
|
7
|
+
end
|
8
|
+
|
9
|
+
def force_flush_for_test!
|
10
|
+
unless Sidekiq::Grouping::Config.tests_env
|
11
|
+
Sidekiq::Grouping.logger.warn(
|
12
|
+
"**************************************************"
|
13
|
+
)
|
14
|
+
Sidekiq::Grouping.logger.warn([
|
15
|
+
"⛔️ force_flush_for_test! for testing API, ",
|
16
|
+
"but this is not the test environment. ",
|
17
|
+
"Please check your environment or ",
|
18
|
+
"change 'tests_env' to cover this one"
|
19
|
+
].join)
|
20
|
+
Sidekiq::Grouping.logger.warn(
|
21
|
+
"**************************************************"
|
22
|
+
)
|
23
|
+
end
|
24
|
+
flush_batches(Sidekiq::Grouping::Batch.all)
|
8
25
|
end
|
9
26
|
|
10
27
|
private
|
11
28
|
|
29
|
+
def flush_batches(batches)
|
30
|
+
batches.compact!
|
31
|
+
flush_concrete(batches)
|
32
|
+
end
|
33
|
+
|
12
34
|
def flush_concrete(batches)
|
13
35
|
return if batches.empty?
|
14
36
|
names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
|
15
37
|
Sidekiq::Grouping.logger.info(
|
16
38
|
"[Sidekiq::Grouping] Trying to flush batched queues: #{names.join(',')}"
|
17
|
-
)
|
39
|
+
) unless Sidekiq::Grouping::Config.tests_env
|
18
40
|
batches.each(&:flush)
|
19
41
|
end
|
20
42
|
end
|
@@ -13,10 +13,10 @@ module Sidekiq
|
|
13
13
|
|
14
14
|
def push_msg(name, msg, remember_unique = false)
|
15
15
|
redis do |conn|
|
16
|
-
conn.multi do
|
17
|
-
|
18
|
-
|
19
|
-
|
16
|
+
conn.multi do |pipeline|
|
17
|
+
pipeline.sadd(ns("batches"), name)
|
18
|
+
pipeline.rpush(ns(name), msg)
|
19
|
+
pipeline.sadd(unique_messages_key(name), msg) if remember_unique
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -32,7 +32,7 @@ module Sidekiq
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def batches
|
35
|
-
redis { |conn| conn.smembers(ns(
|
35
|
+
redis { |conn| conn.smembers(ns("batches")) }
|
36
36
|
end
|
37
37
|
|
38
38
|
def pluck(name, limit)
|
data/lib/sidekiq/grouping/web.rb
CHANGED
@@ -11,11 +11,11 @@ module Sidekiq
|
|
11
11
|
erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
|
12
12
|
end
|
13
13
|
|
14
|
-
app.post "/grouping
|
15
|
-
worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(name)
|
14
|
+
app.post "/grouping/:name/delete" do
|
15
|
+
worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(params['name'])
|
16
16
|
batch = Sidekiq::Grouping::Batch.new(worker_class, queue)
|
17
17
|
batch.delete
|
18
|
-
redirect "#{root_path}
|
18
|
+
redirect "#{root_path}grouping"
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
@@ -25,4 +25,3 @@ end
|
|
25
25
|
|
26
26
|
Sidekiq::Web.register(Sidekiq::Grouping::Web)
|
27
27
|
Sidekiq::Web.tabs["Grouping"] = "grouping"
|
28
|
-
|
data/lib/sidekiq/grouping.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require "active_support"
|
1
2
|
require "active_support/core_ext/string"
|
2
3
|
require "active_support/configurable"
|
3
4
|
require "active_support/core_ext/numeric/time"
|
@@ -19,6 +20,10 @@ module Sidekiq::Grouping
|
|
19
20
|
@logger ||= Sidekiq.logger
|
20
21
|
end
|
21
22
|
|
23
|
+
def force_flush_for_test!
|
24
|
+
Sidekiq::Grouping::Flusher.new.force_flush_for_test!
|
25
|
+
end
|
26
|
+
|
22
27
|
def start!
|
23
28
|
interval = Sidekiq::Grouping::Config.poll_interval
|
24
29
|
@observer = Sidekiq::Grouping::FlusherObserver.new
|
data/sidekiq-grouping.gemspec
CHANGED
@@ -18,12 +18,13 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
-
spec.add_development_dependency "bundler", "
|
21
|
+
spec.add_development_dependency "bundler", "> 1.5"
|
22
22
|
spec.add_development_dependency "rake"
|
23
23
|
spec.add_development_dependency "rspec"
|
24
24
|
spec.add_development_dependency "simplecov"
|
25
25
|
spec.add_development_dependency "rspec-sidekiq"
|
26
26
|
spec.add_development_dependency "timecop"
|
27
|
+
spec.add_development_dependency "appraisal"
|
27
28
|
|
28
29
|
spec.add_dependency "activesupport"
|
29
30
|
spec.add_dependency "sidekiq", ">= 3.4.2"
|
data/spec/modules/batch_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Sidekiq::Grouping::Batch do
|
|
6
6
|
context 'adding' do
|
7
7
|
it 'must enqueue unbatched worker' do
|
8
8
|
RegularWorker.perform_async('bar')
|
9
|
-
expect(RegularWorker).to
|
9
|
+
expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'must not enqueue batched worker' do
|
@@ -67,7 +67,9 @@ describe Sidekiq::Grouping::Batch do
|
|
67
67
|
expect(batch.could_flush?).to be_falsy
|
68
68
|
10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
|
69
69
|
batch.flush
|
70
|
-
expect(BatchedSizeWorker).to
|
70
|
+
expect(BatchedSizeWorker).to(
|
71
|
+
have_enqueued_sidekiq_job([["bar0"], ["bar1"]])
|
72
|
+
)
|
71
73
|
expect(batch.size).to eq(7)
|
72
74
|
end
|
73
75
|
end
|
@@ -124,7 +126,7 @@ describe Sidekiq::Grouping::Batch do
|
|
124
126
|
|
125
127
|
private
|
126
128
|
def expect_batch(klass, queue)
|
127
|
-
expect(klass).to_not
|
129
|
+
expect(klass).to_not have_enqueued_sidekiq_job("bar")
|
128
130
|
batch = subject.new(klass.name, queue)
|
129
131
|
stats = subject.all
|
130
132
|
expect(batch.size).to eq(1)
|
metadata
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-grouping
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Sokolov
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-07-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.5'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.5'
|
27
27
|
- !ruby/object:Gem::Dependency
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: appraisal
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: activesupport
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,7 +150,7 @@ dependencies:
|
|
136
150
|
- - ">="
|
137
151
|
- !ruby/object:Gem::Version
|
138
152
|
version: '0'
|
139
|
-
description:
|
153
|
+
description:
|
140
154
|
email:
|
141
155
|
- gzigzigzeo@gmail.com
|
142
156
|
executables: []
|
@@ -146,10 +160,16 @@ files:
|
|
146
160
|
- ".gitignore"
|
147
161
|
- ".rubocop.yml"
|
148
162
|
- ".travis.yml"
|
163
|
+
- Appraisals
|
149
164
|
- Gemfile
|
150
165
|
- LICENSE.txt
|
151
166
|
- README.md
|
152
167
|
- Rakefile
|
168
|
+
- gemfiles/sidekiq_4.0.gemfile
|
169
|
+
- gemfiles/sidekiq_4.1.gemfile
|
170
|
+
- gemfiles/sidekiq_4.2.gemfile
|
171
|
+
- gemfiles/sidekiq_5.0.gemfile
|
172
|
+
- gemfiles/sidekiq_master.gemfile
|
153
173
|
- lib/sidekiq/grouping.rb
|
154
174
|
- lib/sidekiq/grouping/batch.rb
|
155
175
|
- lib/sidekiq/grouping/config.rb
|
@@ -170,7 +190,7 @@ homepage: http://github.com/gzigzigzeo/sidekiq-grouping
|
|
170
190
|
licenses:
|
171
191
|
- MIT
|
172
192
|
metadata: {}
|
173
|
-
post_install_message:
|
193
|
+
post_install_message:
|
174
194
|
rdoc_options: []
|
175
195
|
require_paths:
|
176
196
|
- lib
|
@@ -185,9 +205,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
185
205
|
- !ruby/object:Gem::Version
|
186
206
|
version: '0'
|
187
207
|
requirements: []
|
188
|
-
|
189
|
-
|
190
|
-
signing_key:
|
208
|
+
rubygems_version: 3.1.6
|
209
|
+
signing_key:
|
191
210
|
specification_version: 4
|
192
211
|
summary: Allows identical sidekiq jobs to be processed with a single background call
|
193
212
|
test_files:
|