sidekiq-grouping 1.0.9 → 1.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +11 -0
- data/Appraisals +19 -0
- data/README.md +51 -2
- data/gemfiles/sidekiq_4.0.gemfile +7 -0
- data/gemfiles/sidekiq_4.1.gemfile +7 -0
- data/gemfiles/sidekiq_4.2.gemfile +7 -0
- data/gemfiles/sidekiq_5.0.gemfile +7 -0
- data/gemfiles/sidekiq_master.gemfile +7 -0
- data/lib/sidekiq/grouping.rb +4 -0
- data/lib/sidekiq/grouping/config.rb +8 -1
- data/lib/sidekiq/grouping/flusher.rb +25 -3
- data/lib/sidekiq/grouping/version.rb +1 -1
- data/sidekiq-grouping.gemspec +1 -0
- data/spec/modules/batch_spec.rb +5 -3
- metadata +23 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fd82358a1dc3ea13dcebc3d113272316e3c334fa3e562dad7eb2828590626988
|
4
|
+
data.tar.gz: 90a9116faf2392b7b5d81ce80346afd3741c897bf60b215eaac43f954e5fcb38
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f08a2fe6ed7811eefae95487143f0db3beb5343c3c3b11b2e7472b805b9eb0a5b6394c84b933219bf2648e6706a463bfc2cd30f1f1e623f48e665d9dec452b0d
|
7
|
+
data.tar.gz: ca99dbc220c9e757549bfd5659ac0673db96f1eadb394b2a396eb7581df482156c83bb107eee98e91744f0f7d93663d393d25f1b7996aac2df84f5e09d69baa3
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -1,7 +1,18 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
3
|
- 2.2
|
4
|
+
- 2.3.1
|
5
|
+
- 2.4
|
6
|
+
- 2.5
|
7
|
+
|
4
8
|
cache: bundler
|
5
9
|
|
6
10
|
services:
|
7
11
|
- redis-server
|
12
|
+
|
13
|
+
gemfile:
|
14
|
+
- gemfiles/sidekiq_4.0.gemfile
|
15
|
+
- gemfiles/sidekiq_4.1.gemfile
|
16
|
+
- gemfiles/sidekiq_4.2.gemfile
|
17
|
+
- gemfiles/sidekiq_5.0.gemfile
|
18
|
+
- gemfiles/sidekiq_master.gemfile
|
data/Appraisals
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
appraise 'sidekiq-4.0' do
|
2
|
+
gem 'sidekiq', '~> 4.0.0'
|
3
|
+
end
|
4
|
+
|
5
|
+
appraise 'sidekiq-4.1' do
|
6
|
+
gem 'sidekiq', '~> 4.1.0'
|
7
|
+
end
|
8
|
+
|
9
|
+
appraise 'sidekiq-4.2' do
|
10
|
+
gem 'sidekiq', '~> 4.2.0'
|
11
|
+
end
|
12
|
+
|
13
|
+
appraise 'sidekiq-5.0' do
|
14
|
+
gem 'sidekiq', '~> 5.0.0'
|
15
|
+
end
|
16
|
+
|
17
|
+
appraise 'sidekiq-master' do
|
18
|
+
gem 'sidekiq', github: 'mperham/sidekiq'
|
19
|
+
end
|
data/README.md
CHANGED
@@ -111,6 +111,8 @@ This jobs will be grouped into the single job with the single argument:
|
|
111
111
|
# => [[5]]
|
112
112
|
```
|
113
113
|
|
114
|
+
- `tests_env` is used to silence some logging in test environments (see below). Default: true if `Rails.env.test?`, false otherwise.
|
115
|
+
|
114
116
|
## Web UI
|
115
117
|
|
116
118
|
![Web UI](web.png)
|
@@ -142,9 +144,56 @@ Sidekiq::Grouping::Config.lock_ttl = 1
|
|
142
144
|
|
143
145
|
Note that you should set poll_interval option inside of sidekiq.yml to take effect. Setting this param in your ruby code won't change actual polling frequency.
|
144
146
|
|
145
|
-
##
|
147
|
+
## Testing with Sidekiq::Testing.fake!
|
148
|
+
|
149
|
+
Sidekiq::Grouping uses internal queues for grouping tasks. If you need to force flush internal queues into normal Sidekiq queues, use `Sidekiq::Grouping.force_flush_for_test!`.
|
150
|
+
|
151
|
+
See example:
|
152
|
+
|
153
|
+
```ruby
|
154
|
+
# worker
|
155
|
+
class GroupedWorker
|
156
|
+
|
157
|
+
include Sidekiq::Worker
|
158
|
+
sidekiq_options(
|
159
|
+
queue: :custom_queue,
|
160
|
+
retry: 5,
|
161
|
+
batch_flush_size: 9,
|
162
|
+
batch_flush_interval: 10,
|
163
|
+
batch_size: 3,
|
164
|
+
batch_unique: true
|
165
|
+
)
|
166
|
+
|
167
|
+
def perform(grouped_arguments)
|
168
|
+
# ... important payload
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
|
173
|
+
# test itself
|
174
|
+
RSpec.describe GroupedWorker, type: :worker do
|
146
175
|
|
147
|
-
|
176
|
+
describe '#perform' do
|
177
|
+
it 'calls perform with array of arguments' do
|
178
|
+
Sidekiq::Testing.fake! do
|
179
|
+
described_class.perform_async(1)
|
180
|
+
described_class.perform_async(1)
|
181
|
+
described_class.perform_async(2)
|
182
|
+
described_class.perform_async(2)
|
183
|
+
|
184
|
+
# All 4 above asks will be put to :custom_queue despite of :batch_flush_size is set to 9.
|
185
|
+
Sidekiq::Grouping.force_flush_for_test!
|
186
|
+
|
187
|
+
last_job = described_class.jobs.last
|
188
|
+
expect(last_job['args']).to eq([[[1], [2]]])
|
189
|
+
expect(last_job['queue']).to eq('custom_queue')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
|
196
|
+
```
|
148
197
|
|
149
198
|
## Installation
|
150
199
|
|
data/lib/sidekiq/grouping.rb
CHANGED
@@ -19,6 +19,10 @@ module Sidekiq::Grouping
|
|
19
19
|
@logger ||= Sidekiq.logger
|
20
20
|
end
|
21
21
|
|
22
|
+
def force_flush_for_test!
|
23
|
+
Sidekiq::Grouping::Flusher.new.force_flush_for_test!
|
24
|
+
end
|
25
|
+
|
22
26
|
def start!
|
23
27
|
interval = Sidekiq::Grouping::Config.poll_interval
|
24
28
|
@observer = Sidekiq::Grouping::FlusherObserver.new
|
@@ -2,7 +2,7 @@ module Sidekiq::Grouping::Config
|
|
2
2
|
include ActiveSupport::Configurable
|
3
3
|
|
4
4
|
def self.options
|
5
|
-
Sidekiq.options["grouping"] || {}
|
5
|
+
Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {} # sidekiq 5.x use symbol in keys
|
6
6
|
end
|
7
7
|
|
8
8
|
# Queue size overflow check polling interval
|
@@ -19,4 +19,11 @@ module Sidekiq::Grouping::Config
|
|
19
19
|
config_accessor :lock_ttl do
|
20
20
|
options[:lock_ttl] || 1
|
21
21
|
end
|
22
|
+
|
23
|
+
# Option to override how Sidekiq::Grouping know about tests env
|
24
|
+
config_accessor :tests_env do
|
25
|
+
options[:tests_env] || (
|
26
|
+
defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
|
27
|
+
)
|
28
|
+
end
|
22
29
|
end
|
@@ -3,18 +3,40 @@ class Sidekiq::Grouping::Flusher
|
|
3
3
|
batches = Sidekiq::Grouping::Batch.all.map do |batch|
|
4
4
|
batch if batch.could_flush?
|
5
5
|
end
|
6
|
-
batches
|
7
|
-
|
6
|
+
flush_batches(batches)
|
7
|
+
end
|
8
|
+
|
9
|
+
def force_flush_for_test!
|
10
|
+
unless Sidekiq::Grouping::Config.tests_env
|
11
|
+
Sidekiq::Grouping.logger.warn(
|
12
|
+
"**************************************************"
|
13
|
+
)
|
14
|
+
Sidekiq::Grouping.logger.warn([
|
15
|
+
"⛔️ force_flush_for_test! for testing API, ",
|
16
|
+
"but this is not the test environment. ",
|
17
|
+
"Please check your environment or ",
|
18
|
+
"change 'tests_env' to cover this one"
|
19
|
+
].join)
|
20
|
+
Sidekiq::Grouping.logger.warn(
|
21
|
+
"**************************************************"
|
22
|
+
)
|
23
|
+
end
|
24
|
+
flush_batches(Sidekiq::Grouping::Batch.all)
|
8
25
|
end
|
9
26
|
|
10
27
|
private
|
11
28
|
|
29
|
+
def flush_batches(batches)
|
30
|
+
batches.compact!
|
31
|
+
flush_concrete(batches)
|
32
|
+
end
|
33
|
+
|
12
34
|
def flush_concrete(batches)
|
13
35
|
return if batches.empty?
|
14
36
|
names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
|
15
37
|
Sidekiq::Grouping.logger.info(
|
16
38
|
"[Sidekiq::Grouping] Trying to flush batched queues: #{names.join(',')}"
|
17
|
-
)
|
39
|
+
) unless Sidekiq::Grouping::Config.tests_env
|
18
40
|
batches.each(&:flush)
|
19
41
|
end
|
20
42
|
end
|
data/sidekiq-grouping.gemspec
CHANGED
@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_development_dependency "simplecov"
|
25
25
|
spec.add_development_dependency "rspec-sidekiq"
|
26
26
|
spec.add_development_dependency "timecop"
|
27
|
+
spec.add_development_dependency "appraisal"
|
27
28
|
|
28
29
|
spec.add_dependency "activesupport"
|
29
30
|
spec.add_dependency "sidekiq", ">= 3.4.2"
|
data/spec/modules/batch_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Sidekiq::Grouping::Batch do
|
|
6
6
|
context 'adding' do
|
7
7
|
it 'must enqueue unbatched worker' do
|
8
8
|
RegularWorker.perform_async('bar')
|
9
|
-
expect(RegularWorker).to
|
9
|
+
expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'must not enqueue batched worker' do
|
@@ -67,7 +67,9 @@ describe Sidekiq::Grouping::Batch do
|
|
67
67
|
expect(batch.could_flush?).to be_falsy
|
68
68
|
10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
|
69
69
|
batch.flush
|
70
|
-
expect(BatchedSizeWorker).to
|
70
|
+
expect(BatchedSizeWorker).to(
|
71
|
+
have_enqueued_sidekiq_job([["bar0"], ["bar1"]])
|
72
|
+
)
|
71
73
|
expect(batch.size).to eq(7)
|
72
74
|
end
|
73
75
|
end
|
@@ -124,7 +126,7 @@ describe Sidekiq::Grouping::Batch do
|
|
124
126
|
|
125
127
|
private
|
126
128
|
def expect_batch(klass, queue)
|
127
|
-
expect(klass).to_not
|
129
|
+
expect(klass).to_not have_enqueued_sidekiq_job("bar")
|
128
130
|
batch = subject.new(klass.name, queue)
|
129
131
|
stats = subject.all
|
130
132
|
expect(batch.size).to eq(1)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-grouping
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Sokolov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: appraisal
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: activesupport
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -146,10 +160,16 @@ files:
|
|
146
160
|
- ".gitignore"
|
147
161
|
- ".rubocop.yml"
|
148
162
|
- ".travis.yml"
|
163
|
+
- Appraisals
|
149
164
|
- Gemfile
|
150
165
|
- LICENSE.txt
|
151
166
|
- README.md
|
152
167
|
- Rakefile
|
168
|
+
- gemfiles/sidekiq_4.0.gemfile
|
169
|
+
- gemfiles/sidekiq_4.1.gemfile
|
170
|
+
- gemfiles/sidekiq_4.2.gemfile
|
171
|
+
- gemfiles/sidekiq_5.0.gemfile
|
172
|
+
- gemfiles/sidekiq_master.gemfile
|
153
173
|
- lib/sidekiq/grouping.rb
|
154
174
|
- lib/sidekiq/grouping/batch.rb
|
155
175
|
- lib/sidekiq/grouping/config.rb
|
@@ -186,7 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
186
206
|
version: '0'
|
187
207
|
requirements: []
|
188
208
|
rubyforge_project:
|
189
|
-
rubygems_version: 2.
|
209
|
+
rubygems_version: 2.7.3
|
190
210
|
signing_key:
|
191
211
|
specification_version: 4
|
192
212
|
summary: Allows identical sidekiq jobs to be processed with a single background call
|