sidekiq-grouping 1.0.9 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +11 -0
- data/Appraisals +19 -0
- data/README.md +51 -2
- data/gemfiles/sidekiq_4.0.gemfile +7 -0
- data/gemfiles/sidekiq_4.1.gemfile +7 -0
- data/gemfiles/sidekiq_4.2.gemfile +7 -0
- data/gemfiles/sidekiq_5.0.gemfile +7 -0
- data/gemfiles/sidekiq_master.gemfile +7 -0
- data/lib/sidekiq/grouping.rb +4 -0
- data/lib/sidekiq/grouping/config.rb +8 -1
- data/lib/sidekiq/grouping/flusher.rb +25 -3
- data/lib/sidekiq/grouping/version.rb +1 -1
- data/sidekiq-grouping.gemspec +1 -0
- data/spec/modules/batch_spec.rb +5 -3
- metadata +23 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fd82358a1dc3ea13dcebc3d113272316e3c334fa3e562dad7eb2828590626988
|
4
|
+
data.tar.gz: 90a9116faf2392b7b5d81ce80346afd3741c897bf60b215eaac43f954e5fcb38
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f08a2fe6ed7811eefae95487143f0db3beb5343c3c3b11b2e7472b805b9eb0a5b6394c84b933219bf2648e6706a463bfc2cd30f1f1e623f48e665d9dec452b0d
|
7
|
+
data.tar.gz: ca99dbc220c9e757549bfd5659ac0673db96f1eadb394b2a396eb7581df482156c83bb107eee98e91744f0f7d93663d393d25f1b7996aac2df84f5e09d69baa3
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -1,7 +1,18 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
3
|
- 2.2
|
4
|
+
- 2.3.1
|
5
|
+
- 2.4
|
6
|
+
- 2.5
|
7
|
+
|
4
8
|
cache: bundler
|
5
9
|
|
6
10
|
services:
|
7
11
|
- redis-server
|
12
|
+
|
13
|
+
gemfile:
|
14
|
+
- gemfiles/sidekiq_4.0.gemfile
|
15
|
+
- gemfiles/sidekiq_4.1.gemfile
|
16
|
+
- gemfiles/sidekiq_4.2.gemfile
|
17
|
+
- gemfiles/sidekiq_5.0.gemfile
|
18
|
+
- gemfiles/sidekiq_master.gemfile
|
data/Appraisals
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
appraise 'sidekiq-4.0' do
|
2
|
+
gem 'sidekiq', '~> 4.0.0'
|
3
|
+
end
|
4
|
+
|
5
|
+
appraise 'sidekiq-4.1' do
|
6
|
+
gem 'sidekiq', '~> 4.1.0'
|
7
|
+
end
|
8
|
+
|
9
|
+
appraise 'sidekiq-4.2' do
|
10
|
+
gem 'sidekiq', '~> 4.2.0'
|
11
|
+
end
|
12
|
+
|
13
|
+
appraise 'sidekiq-5.0' do
|
14
|
+
gem 'sidekiq', '~> 5.0.0'
|
15
|
+
end
|
16
|
+
|
17
|
+
appraise 'sidekiq-master' do
|
18
|
+
gem 'sidekiq', github: 'mperham/sidekiq'
|
19
|
+
end
|
data/README.md
CHANGED
@@ -111,6 +111,8 @@ This jobs will be grouped into the single job with the single argument:
|
|
111
111
|
# => [[5]]
|
112
112
|
```
|
113
113
|
|
114
|
+
- `tests_env` is used to silence some logging in test environments (see below). Default: true if `Rails.env.test?`, false otherwise.
|
115
|
+
|
114
116
|
## Web UI
|
115
117
|
|
116
118
|

|
@@ -142,9 +144,56 @@ Sidekiq::Grouping::Config.lock_ttl = 1
|
|
142
144
|
|
143
145
|
Note that you should set poll_interval option inside of sidekiq.yml to take effect. Setting this param in your ruby code won't change actual polling frequency.
|
144
146
|
|
145
|
-
##
|
147
|
+
## Testing with Sidekiq::Testing.fake!
|
148
|
+
|
149
|
+
Sidekiq::Grouping uses internal queues for grouping tasks. If you need to force flush internal queues into normal Sidekiq queues, use `Sidekiq::Grouping.force_flush_for_test!`.
|
150
|
+
|
151
|
+
See example:
|
152
|
+
|
153
|
+
```ruby
|
154
|
+
# worker
|
155
|
+
class GroupedWorker
|
156
|
+
|
157
|
+
include Sidekiq::Worker
|
158
|
+
sidekiq_options(
|
159
|
+
queue: :custom_queue,
|
160
|
+
retry: 5,
|
161
|
+
batch_flush_size: 9,
|
162
|
+
batch_flush_interval: 10,
|
163
|
+
batch_size: 3,
|
164
|
+
batch_unique: true
|
165
|
+
)
|
166
|
+
|
167
|
+
def perform(grouped_arguments)
|
168
|
+
# ... important payload
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
|
173
|
+
# test itself
|
174
|
+
RSpec.describe GroupedWorker, type: :worker do
|
146
175
|
|
147
|
-
|
176
|
+
describe '#perform' do
|
177
|
+
it 'calls perform with array of arguments' do
|
178
|
+
Sidekiq::Testing.fake! do
|
179
|
+
described_class.perform_async(1)
|
180
|
+
described_class.perform_async(1)
|
181
|
+
described_class.perform_async(2)
|
182
|
+
described_class.perform_async(2)
|
183
|
+
|
184
|
+
# All 4 above asks will be put to :custom_queue despite of :batch_flush_size is set to 9.
|
185
|
+
Sidekiq::Grouping.force_flush_for_test!
|
186
|
+
|
187
|
+
last_job = described_class.jobs.last
|
188
|
+
expect(last_job['args']).to eq([[[1], [2]]])
|
189
|
+
expect(last_job['queue']).to eq('custom_queue')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
|
196
|
+
```
|
148
197
|
|
149
198
|
## Installation
|
150
199
|
|
data/lib/sidekiq/grouping.rb
CHANGED
@@ -19,6 +19,10 @@ module Sidekiq::Grouping
|
|
19
19
|
@logger ||= Sidekiq.logger
|
20
20
|
end
|
21
21
|
|
22
|
+
def force_flush_for_test!
|
23
|
+
Sidekiq::Grouping::Flusher.new.force_flush_for_test!
|
24
|
+
end
|
25
|
+
|
22
26
|
def start!
|
23
27
|
interval = Sidekiq::Grouping::Config.poll_interval
|
24
28
|
@observer = Sidekiq::Grouping::FlusherObserver.new
|
@@ -2,7 +2,7 @@ module Sidekiq::Grouping::Config
|
|
2
2
|
include ActiveSupport::Configurable
|
3
3
|
|
4
4
|
def self.options
|
5
|
-
Sidekiq.options["grouping"] || {}
|
5
|
+
Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {} # sidekiq 5.x use symbol in keys
|
6
6
|
end
|
7
7
|
|
8
8
|
# Queue size overflow check polling interval
|
@@ -19,4 +19,11 @@ module Sidekiq::Grouping::Config
|
|
19
19
|
config_accessor :lock_ttl do
|
20
20
|
options[:lock_ttl] || 1
|
21
21
|
end
|
22
|
+
|
23
|
+
# Option to override how Sidekiq::Grouping know about tests env
|
24
|
+
config_accessor :tests_env do
|
25
|
+
options[:tests_env] || (
|
26
|
+
defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
|
27
|
+
)
|
28
|
+
end
|
22
29
|
end
|
@@ -3,18 +3,40 @@ class Sidekiq::Grouping::Flusher
|
|
3
3
|
batches = Sidekiq::Grouping::Batch.all.map do |batch|
|
4
4
|
batch if batch.could_flush?
|
5
5
|
end
|
6
|
-
batches
|
7
|
-
|
6
|
+
flush_batches(batches)
|
7
|
+
end
|
8
|
+
|
9
|
+
def force_flush_for_test!
|
10
|
+
unless Sidekiq::Grouping::Config.tests_env
|
11
|
+
Sidekiq::Grouping.logger.warn(
|
12
|
+
"**************************************************"
|
13
|
+
)
|
14
|
+
Sidekiq::Grouping.logger.warn([
|
15
|
+
"⛔️ force_flush_for_test! for testing API, ",
|
16
|
+
"but this is not the test environment. ",
|
17
|
+
"Please check your environment or ",
|
18
|
+
"change 'tests_env' to cover this one"
|
19
|
+
].join)
|
20
|
+
Sidekiq::Grouping.logger.warn(
|
21
|
+
"**************************************************"
|
22
|
+
)
|
23
|
+
end
|
24
|
+
flush_batches(Sidekiq::Grouping::Batch.all)
|
8
25
|
end
|
9
26
|
|
10
27
|
private
|
11
28
|
|
29
|
+
def flush_batches(batches)
|
30
|
+
batches.compact!
|
31
|
+
flush_concrete(batches)
|
32
|
+
end
|
33
|
+
|
12
34
|
def flush_concrete(batches)
|
13
35
|
return if batches.empty?
|
14
36
|
names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
|
15
37
|
Sidekiq::Grouping.logger.info(
|
16
38
|
"[Sidekiq::Grouping] Trying to flush batched queues: #{names.join(',')}"
|
17
|
-
)
|
39
|
+
) unless Sidekiq::Grouping::Config.tests_env
|
18
40
|
batches.each(&:flush)
|
19
41
|
end
|
20
42
|
end
|
data/sidekiq-grouping.gemspec
CHANGED
@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_development_dependency "simplecov"
|
25
25
|
spec.add_development_dependency "rspec-sidekiq"
|
26
26
|
spec.add_development_dependency "timecop"
|
27
|
+
spec.add_development_dependency "appraisal"
|
27
28
|
|
28
29
|
spec.add_dependency "activesupport"
|
29
30
|
spec.add_dependency "sidekiq", ">= 3.4.2"
|
data/spec/modules/batch_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Sidekiq::Grouping::Batch do
|
|
6
6
|
context 'adding' do
|
7
7
|
it 'must enqueue unbatched worker' do
|
8
8
|
RegularWorker.perform_async('bar')
|
9
|
-
expect(RegularWorker).to
|
9
|
+
expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'must not enqueue batched worker' do
|
@@ -67,7 +67,9 @@ describe Sidekiq::Grouping::Batch do
|
|
67
67
|
expect(batch.could_flush?).to be_falsy
|
68
68
|
10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
|
69
69
|
batch.flush
|
70
|
-
expect(BatchedSizeWorker).to
|
70
|
+
expect(BatchedSizeWorker).to(
|
71
|
+
have_enqueued_sidekiq_job([["bar0"], ["bar1"]])
|
72
|
+
)
|
71
73
|
expect(batch.size).to eq(7)
|
72
74
|
end
|
73
75
|
end
|
@@ -124,7 +126,7 @@ describe Sidekiq::Grouping::Batch do
|
|
124
126
|
|
125
127
|
private
|
126
128
|
def expect_batch(klass, queue)
|
127
|
-
expect(klass).to_not
|
129
|
+
expect(klass).to_not have_enqueued_sidekiq_job("bar")
|
128
130
|
batch = subject.new(klass.name, queue)
|
129
131
|
stats = subject.all
|
130
132
|
expect(batch.size).to eq(1)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-grouping
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Sokolov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: appraisal
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: activesupport
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -146,10 +160,16 @@ files:
|
|
146
160
|
- ".gitignore"
|
147
161
|
- ".rubocop.yml"
|
148
162
|
- ".travis.yml"
|
163
|
+
- Appraisals
|
149
164
|
- Gemfile
|
150
165
|
- LICENSE.txt
|
151
166
|
- README.md
|
152
167
|
- Rakefile
|
168
|
+
- gemfiles/sidekiq_4.0.gemfile
|
169
|
+
- gemfiles/sidekiq_4.1.gemfile
|
170
|
+
- gemfiles/sidekiq_4.2.gemfile
|
171
|
+
- gemfiles/sidekiq_5.0.gemfile
|
172
|
+
- gemfiles/sidekiq_master.gemfile
|
153
173
|
- lib/sidekiq/grouping.rb
|
154
174
|
- lib/sidekiq/grouping/batch.rb
|
155
175
|
- lib/sidekiq/grouping/config.rb
|
@@ -186,7 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
186
206
|
version: '0'
|
187
207
|
requirements: []
|
188
208
|
rubyforge_project:
|
189
|
-
rubygems_version: 2.
|
209
|
+
rubygems_version: 2.7.3
|
190
210
|
signing_key:
|
191
211
|
specification_version: 4
|
192
212
|
summary: Allows identical sidekiq jobs to be processed with a single background call
|