sidekiq-clutch 2.0.0 → 2.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -0
- data/.gitignore +1 -0
- data/CHANGELOG.md +22 -0
- data/lib/sidekiq/clutch/job_wrapper.rb +1 -4
- data/lib/sidekiq/clutch/jobs_collection.rb +3 -5
- data/lib/sidekiq/clutch/version.rb +1 -1
- data/lib/sidekiq/clutch/worker.rb +1 -1
- data/lib/sidekiq/clutch.rb +72 -12
- data/sidekiq-clutch.gemspec +4 -5
- metadata +12 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8a79d553989453b6f6bdb859e2c6a0484ed1d3fdc6992dfa0aa80b79810fa4c2
|
4
|
+
data.tar.gz: 06d6490367f6ed9ad7d1e616128f6a6498242637786e66d8724bea4735718f89
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7a80caa3060acfcdb7f5cb3c4497261030c4f07cc1d528bbe078e6c3bb434117ad031d342f3940eaab59de9afb3b39d117a516933d92d3dd17475bea3c56d10d
|
7
|
+
data.tar.gz: ad2c11c85514d5cace032bace89031d2994be6cad8533bffaf1d498d869bdea9eb1dbb4f157e93c393b9ec3b72e3bbfebdb06d3fdc099ea02d41a032d98ecb5d
|
data/.circleci/config.yml
CHANGED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,25 @@
|
|
1
|
+
# 2.1.1 - Sep 14, 2021
|
2
|
+
|
3
|
+
* FIX: Call correct clean up method when legacy jobs are finished
|
4
|
+
|
5
|
+
# 2.1.0 - Sep 10, 2021
|
6
|
+
|
7
|
+
* PERF: Don't pass step data in success callback to every batch
|
8
|
+
|
9
|
+
NOTE: This changes the structure of arguments passed to Batch callbacks, though we
|
10
|
+
also handle the old way of doing it (v2.0.2 and before) so it shouldn't affect
|
11
|
+
any in-progress batches you may have running.
|
12
|
+
|
13
|
+
# 2.0.2 - Sep 24, 2020
|
14
|
+
|
15
|
+
* PERF: Delete keys based on known values, instead of a glob
|
16
|
+
* PERF: Don't bother enqueing a `:complete` callback if no `on_failure` is specified
|
17
|
+
|
18
|
+
# 2.0.1 - Jun 1, 2020
|
19
|
+
|
20
|
+
* CHORE: Bump development dependency versions
|
21
|
+
* CHORE: Add .ruby-version file for easier development
|
22
|
+
|
1
23
|
# 2.0.0 - Feb 5, 2020
|
2
24
|
|
3
25
|
* BREAKING: Treat each parallel block as a distinct step
|
@@ -3,9 +3,6 @@ module Sidekiq
|
|
3
3
|
class JobWrapper
|
4
4
|
include Sidekiq::Worker
|
5
5
|
|
6
|
-
# 22 days - how long a Sidekiq job can live with exponential backoff
|
7
|
-
RESULT_KEY_EXPIRATION_DURATION = 22 * 24 * 60 * 60
|
8
|
-
|
9
6
|
def perform(bid, job_class, args, last_result_key, current_result_key)
|
10
7
|
job = Object.const_get(job_class).new
|
11
8
|
assign_previous_results(job, last_result_key)
|
@@ -14,7 +11,7 @@ module Sidekiq
|
|
14
11
|
Sidekiq.redis do |redis|
|
15
12
|
redis.multi do |multi|
|
16
13
|
multi.rpush(current_result_key, result.to_json)
|
17
|
-
multi.expire(current_result_key,
|
14
|
+
multi.expire(current_result_key, TEMPORARY_KEY_EXPIRATION_DURATION)
|
18
15
|
end
|
19
16
|
end
|
20
17
|
end
|
@@ -4,7 +4,6 @@ module Sidekiq
|
|
4
4
|
def initialize(service)
|
5
5
|
@service = service
|
6
6
|
@jobs = []
|
7
|
-
@result_key_prefix = SecureRandom.uuid
|
8
7
|
@result_key_index = 0
|
9
8
|
end
|
10
9
|
|
@@ -25,19 +24,18 @@ module Sidekiq
|
|
25
24
|
@jobs << new_parallel_step unless continue_existing_parallel_step?
|
26
25
|
@jobs.last['parallel'] << [klass.name, params]
|
27
26
|
else
|
28
|
-
@jobs << { 'series' => [klass.name, params], '
|
27
|
+
@jobs << { 'series' => [klass.name, params], 'result_key_index' => next_result_key_index }
|
29
28
|
end
|
30
29
|
end
|
31
30
|
|
32
|
-
def
|
31
|
+
def next_result_key_index
|
33
32
|
@result_key_index += 1
|
34
|
-
"#{@result_key_prefix}-#{@result_key_index}"
|
35
33
|
end
|
36
34
|
|
37
35
|
private
|
38
36
|
|
39
37
|
def new_parallel_step
|
40
|
-
{ 'parallel' => [], '
|
38
|
+
{ 'parallel' => [], 'result_key_index' => next_result_key_index, 'parallel_key' => @service.parallel_key }
|
41
39
|
end
|
42
40
|
|
43
41
|
def continue_existing_parallel_step?
|
@@ -15,7 +15,7 @@ module Sidekiq
|
|
15
15
|
Sidekiq.redis do |redis|
|
16
16
|
redis.multi do |multi|
|
17
17
|
multi.rpush(current_result_key, result.to_json)
|
18
|
-
multi.expire(current_result_key,
|
18
|
+
multi.expire(current_result_key, TEMPORARY_KEY_EXPIRATION_DURATION)
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
data/lib/sidekiq/clutch.rb
CHANGED
@@ -6,6 +6,9 @@ require 'sidekiq/clutch/job_wrapper'
|
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
class Clutch
|
9
|
+
# 22 days - how long a Sidekiq job can live with exponential backoff
|
10
|
+
TEMPORARY_KEY_EXPIRATION_DURATION = 22 * 24 * 60 * 60
|
11
|
+
|
9
12
|
def initialize(batch = nil)
|
10
13
|
@batch = batch || Sidekiq::Batch.new
|
11
14
|
end
|
@@ -51,10 +54,12 @@ module Sidekiq
|
|
51
54
|
def setup_batch
|
52
55
|
jobs_queue = jobs.raw.dup
|
53
56
|
step = jobs_queue.shift
|
57
|
+
set_jobs_data_in_redis(jobs_queue)
|
54
58
|
return if step.nil?
|
55
59
|
batch.callback_queue = queue if queue
|
56
|
-
batch.on(:success, Sidekiq::Clutch, '
|
57
|
-
|
60
|
+
batch.on(:success, Sidekiq::Clutch, 'key_base' => key_base, 'result_key_index' => result_key_index(step))
|
61
|
+
on_failure_name = on_failure&.name
|
62
|
+
batch.on(:complete, Sidekiq::Clutch, 'on_failure' => on_failure_name) if on_failure_name
|
58
63
|
batch.jobs do
|
59
64
|
if step['series']
|
60
65
|
series_step(step)
|
@@ -67,14 +72,23 @@ module Sidekiq
|
|
67
72
|
end
|
68
73
|
|
69
74
|
def on_success(status, options)
|
70
|
-
if options['jobs']
|
71
|
-
|
75
|
+
return on_success_legacy(status, options) if options['jobs']
|
76
|
+
|
77
|
+
raise 'invariant: key_base is missing!' unless options['key_base']
|
78
|
+
raise 'invariant: result_key_index is missing!' unless options['result_key_index']
|
79
|
+
|
80
|
+
# NOTE: This is a brand new instance of Sidekiq::Clutch that Sidekiq instantiates,
|
81
|
+
# so we need to set @key_base again.
|
82
|
+
@key_base = options['key_base']
|
83
|
+
remaining_jobs = JSON.parse(Sidekiq.redis { |r| r.get(jobs_key) })
|
84
|
+
if remaining_jobs.empty?
|
85
|
+
clean_up_temporary_keys
|
72
86
|
return
|
73
87
|
end
|
74
88
|
parent_batch = Sidekiq::Batch.new(status.parent_bid)
|
75
89
|
service = self.class.new(parent_batch)
|
76
|
-
service.jobs.raw =
|
77
|
-
service.current_result_key = options['
|
90
|
+
service.jobs.raw = remaining_jobs
|
91
|
+
service.current_result_key = "#{key_base}-#{options['result_key_index']}"
|
78
92
|
service.engage
|
79
93
|
end
|
80
94
|
|
@@ -86,19 +100,61 @@ module Sidekiq
|
|
86
100
|
|
87
101
|
private
|
88
102
|
|
103
|
+
# accept old style of passing job data, will be removed in 3.0
|
104
|
+
def on_success_legacy(status, options)
|
105
|
+
@key_base = options['result_key'].sub(/-\d+$/, '')
|
106
|
+
if options['jobs'].empty?
|
107
|
+
clean_up_temporary_keys
|
108
|
+
return
|
109
|
+
end
|
110
|
+
parent_batch = Sidekiq::Batch.new(status.parent_bid)
|
111
|
+
service = self.class.new(parent_batch)
|
112
|
+
service.jobs.raw = options['jobs']
|
113
|
+
service.current_result_key = options['result_key']
|
114
|
+
service.engage
|
115
|
+
end
|
116
|
+
|
117
|
+
def key_base
|
118
|
+
@key_base ||= SecureRandom.uuid
|
119
|
+
end
|
120
|
+
|
121
|
+
def jobs_key
|
122
|
+
"#{key_base}-jobs"
|
123
|
+
end
|
124
|
+
|
125
|
+
def set_jobs_data_in_redis(data)
|
126
|
+
Sidekiq.redis do |redis|
|
127
|
+
redis.multi do |multi|
|
128
|
+
multi.set(jobs_key, data.to_json)
|
129
|
+
multi.expire(jobs_key, TEMPORARY_KEY_EXPIRATION_DURATION)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
89
134
|
def series_step(step)
|
90
135
|
(klass, params) = step['series']
|
91
|
-
enqueue_job(klass, params, step
|
136
|
+
enqueue_job(klass, params, result_key_index(step))
|
92
137
|
end
|
93
138
|
|
94
139
|
def parallel_step(step)
|
95
140
|
step['parallel'].each do |(klass, params)|
|
96
|
-
enqueue_job(klass, params, step
|
141
|
+
enqueue_job(klass, params, result_key_index(step))
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def result_key_index(step)
|
146
|
+
if step['result_key_index']
|
147
|
+
step['result_key_index']
|
148
|
+
elsif step['result_key'] # legacy style, will be removed in 3.0
|
149
|
+
step['result_key'].split('-').last.to_i
|
150
|
+
else
|
151
|
+
raise "invariant: expected result_key_index passed in step; got: #{step.inspect}"
|
97
152
|
end
|
98
153
|
end
|
99
154
|
|
100
|
-
def enqueue_job(klass, params,
|
155
|
+
def enqueue_job(klass, params, result_key_index)
|
101
156
|
job_options = Object.const_get(klass).sidekiq_options
|
157
|
+
result_key = "#{key_base}-#{result_key_index}"
|
102
158
|
options = {
|
103
159
|
'class' => JobWrapper,
|
104
160
|
'queue' => queue || job_options['queue'],
|
@@ -110,10 +166,14 @@ module Sidekiq
|
|
110
166
|
Sidekiq::Client.push(options)
|
111
167
|
end
|
112
168
|
|
113
|
-
def
|
169
|
+
def clean_up_temporary_keys
|
114
170
|
Sidekiq.redis do |redis|
|
115
|
-
redis.
|
116
|
-
|
171
|
+
redis.del(jobs_key)
|
172
|
+
result_key_index = 1
|
173
|
+
loop do
|
174
|
+
result = redis.del("#{key_base}-#{result_key_index}")
|
175
|
+
result_key_index += 1
|
176
|
+
break if result == 0
|
117
177
|
end
|
118
178
|
end
|
119
179
|
end
|
data/sidekiq-clutch.gemspec
CHANGED
@@ -15,14 +15,13 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.license = 'MIT'
|
16
16
|
|
17
17
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
18
|
-
`git ls-files`.split("\n").reject { |f| f.match(%r{^(
|
18
|
+
`git ls-files`.split("\n").reject { |f| f.match(%r{^(spec|\.ruby-version)}) }
|
19
19
|
end
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
22
|
spec.add_dependency 'sidekiq', '>= 5.0.0'
|
23
23
|
|
24
|
-
spec.add_development_dependency '
|
25
|
-
spec.add_development_dependency '
|
26
|
-
spec.add_development_dependency '
|
27
|
-
spec.add_development_dependency 'rspec', '~> 3.0'
|
24
|
+
spec.add_development_dependency 'bundler', '~> 2.2.4'
|
25
|
+
spec.add_development_dependency 'rake', '~> 13.0.1'
|
26
|
+
spec.add_development_dependency 'rspec', '~> 3.9.0'
|
28
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-clutch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tim Morgan
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -24,62 +24,48 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 5.0.0
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: pry
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: bundler
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
31
|
- - "~>"
|
46
32
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
33
|
+
version: 2.2.4
|
48
34
|
type: :development
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
38
|
- - "~>"
|
53
39
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
40
|
+
version: 2.2.4
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: rake
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
45
|
- - "~>"
|
60
46
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
47
|
+
version: 13.0.1
|
62
48
|
type: :development
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
52
|
- - "~>"
|
67
53
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
54
|
+
version: 13.0.1
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: rspec
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
72
58
|
requirements:
|
73
59
|
- - "~>"
|
74
60
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
61
|
+
version: 3.9.0
|
76
62
|
type: :development
|
77
63
|
prerelease: false
|
78
64
|
version_requirements: !ruby/object:Gem::Requirement
|
79
65
|
requirements:
|
80
66
|
- - "~>"
|
81
67
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
68
|
+
version: 3.9.0
|
83
69
|
description: Sidekiq::Clutch provides an ergonomic wrapper API for Sidekiq Batches
|
84
70
|
so you can easily manage serial and parallel jobs.
|
85
71
|
email:
|
@@ -107,7 +93,7 @@ homepage: https://github.com/seven1m/sidekiq-clutch
|
|
107
93
|
licenses:
|
108
94
|
- MIT
|
109
95
|
metadata: {}
|
110
|
-
post_install_message:
|
96
|
+
post_install_message:
|
111
97
|
rdoc_options: []
|
112
98
|
require_paths:
|
113
99
|
- lib
|
@@ -122,8 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
122
108
|
- !ruby/object:Gem::Version
|
123
109
|
version: '0'
|
124
110
|
requirements: []
|
125
|
-
rubygems_version: 3.
|
126
|
-
signing_key:
|
111
|
+
rubygems_version: 3.1.6
|
112
|
+
signing_key:
|
127
113
|
specification_version: 4
|
128
114
|
summary: An ergonomic wrapper API for Sidekiq Batches
|
129
115
|
test_files: []
|