sidekiq-unique-jobs 7.1.18 → 7.1.21
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -2
- data/README.md +3 -1
- data/lib/sidekiq_unique_jobs/exceptions.rb +3 -3
- data/lib/sidekiq_unique_jobs/lock_args.rb +18 -14
- data/lib/sidekiq_unique_jobs/lock_config.rb +4 -4
- data/lib/sidekiq_unique_jobs/lock_digest.rb +6 -6
- data/lib/sidekiq_unique_jobs/lock_timeout.rb +4 -4
- data/lib/sidekiq_unique_jobs/lock_ttl.rb +4 -4
- data/lib/sidekiq_unique_jobs/middleware.rb +4 -4
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +3 -3
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +3 -3
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +1 -14
- data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +28 -4
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +33 -24
- data/lib/sidekiq_unique_jobs/timer_task.rb +275 -60
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/tasks/changelog.rake +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a009df15de36d139adf211d9c4cd96fe42e8bdfd243831dc7d4f64bc4f98cd56
|
4
|
+
data.tar.gz: 767f9e88f3ca9fe563d34243ee98c1bc6dcb9852575b0e62153a66b213fef50a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c5c4d0e50fea0faac52c9183db15db1121af9e2839996e5593b4404d1cc5d1c50bee60e574d2538149a5a5fbe2832296041524f175a42859d239b1b8fe049165
|
7
|
+
data.tar.gz: df0259ed2f86592b941a1e966926c21f071e2d1bc059379c68270b5878504a118108336be836fb28a4ba6380f1d44a1da890d45c5df8b7dfabede27b49d90105
|
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,29 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
## [
|
3
|
+
## [v7.1.20](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.20) (2022-04-22)
|
4
4
|
|
5
|
-
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.
|
5
|
+
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.19...v7.1.20)
|
6
|
+
|
7
|
+
**Implemented enhancements:**
|
8
|
+
|
9
|
+
- Manually handle timeouts [\#706](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/706) ([mhenrixon](https://github.com/mhenrixon))
|
10
|
+
|
11
|
+
**Merged pull requests:**
|
12
|
+
|
13
|
+
- improve README wrt. middleware config [\#704](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/704) ([slhck](https://github.com/slhck))
|
14
|
+
|
15
|
+
## [v7.1.19](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.19) (2022-04-09)
|
16
|
+
|
17
|
+
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.18...v7.1.19)
|
18
|
+
|
19
|
+
**Fixed bugs:**
|
20
|
+
|
21
|
+
- concurrent-ruby 1.1.10 spikes volume of jobs [\#701](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/701)
|
22
|
+
- Reimplement the entire TimerTask as it was [\#702](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/702) ([mhenrixon](https://github.com/mhenrixon))
|
23
|
+
|
24
|
+
## [v7.1.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.18) (2022-04-05)
|
25
|
+
|
26
|
+
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.17...v7.1.18)
|
6
27
|
|
7
28
|
**Implemented enhancements:**
|
8
29
|
|
data/README.md
CHANGED
@@ -123,9 +123,11 @@ Before v7, the middleware was configured automatically. Since some people report
|
|
123
123
|
|
124
124
|
*NOTE* if you want to use the reaper you also need to configure the server middleware.
|
125
125
|
|
126
|
-
[
|
126
|
+
The following shows how to modify your `config/initializers/sidekiq.rb` file to use the middleware. [Here is a full example.](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/myapp/config/initializers/sidekiq.rb#L12)
|
127
127
|
|
128
128
|
```ruby
|
129
|
+
require "sidekiq-unique-jobs"
|
130
|
+
|
129
131
|
Sidekiq.configure_server do |config|
|
130
132
|
config.redis = { url: ENV["REDIS_URL"], driver: :hiredis }
|
131
133
|
|
@@ -72,14 +72,14 @@ module SidekiqUniqueJobs
|
|
72
72
|
class InvalidUniqueArguments < UniqueJobsError
|
73
73
|
def initialize(options)
|
74
74
|
given = options[:given]
|
75
|
-
|
75
|
+
job_class = options[:job_class]
|
76
76
|
lock_args_method = options[:lock_args_method]
|
77
|
-
lock_args_meth =
|
77
|
+
lock_args_meth = job_class.method(lock_args_method)
|
78
78
|
num_args = lock_args_meth.arity
|
79
79
|
source_location = lock_args_meth.source_location
|
80
80
|
|
81
81
|
super(
|
82
|
-
"#{
|
82
|
+
"#{job_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
|
83
83
|
"\n\n" \
|
84
84
|
" #{source_location.join(':')}"
|
85
85
|
)
|
@@ -26,9 +26,9 @@ module SidekiqUniqueJobs
|
|
26
26
|
|
27
27
|
# @param [Hash] item a Sidekiq job hash
|
28
28
|
def initialize(item)
|
29
|
-
@item
|
30
|
-
@
|
31
|
-
|
29
|
+
@item = item
|
30
|
+
@args = item[ARGS]
|
31
|
+
self.job_class = item[CLASS]
|
32
32
|
end
|
33
33
|
|
34
34
|
# The unique arguments to use for creating a lock
|
@@ -83,31 +83,31 @@ module SidekiqUniqueJobs
|
|
83
83
|
|
84
84
|
# Filters unique arguments by method configured in the sidekiq worker
|
85
85
|
# @param [Array] args the arguments passed to the sidekiq worker
|
86
|
-
# @return [Array] unfiltered unless {#
|
86
|
+
# @return [Array] unfiltered unless {#job_method_defined?}
|
87
87
|
# @return [Array] with the filtered arguments
|
88
88
|
def filter_by_symbol(args)
|
89
|
-
return args unless
|
89
|
+
return args unless job_method_defined?(lock_args_method)
|
90
90
|
|
91
|
-
|
91
|
+
job_class.send(lock_args_method, args)
|
92
92
|
rescue ArgumentError
|
93
93
|
raise SidekiqUniqueJobs::InvalidUniqueArguments,
|
94
94
|
given: args,
|
95
|
-
|
95
|
+
job_class: job_class,
|
96
96
|
lock_args_method: lock_args_method
|
97
97
|
end
|
98
98
|
|
99
99
|
# The method to use for filtering unique arguments
|
100
100
|
def lock_args_method
|
101
|
-
@lock_args_method ||=
|
102
|
-
@lock_args_method ||= :lock_args if
|
103
|
-
@lock_args_method ||= :unique_args if
|
101
|
+
@lock_args_method ||= job_options.slice(LOCK_ARGS_METHOD, UNIQUE_ARGS_METHOD).values.first
|
102
|
+
@lock_args_method ||= :lock_args if job_method_defined?(:lock_args)
|
103
|
+
@lock_args_method ||= :unique_args if job_method_defined?(:unique_args)
|
104
104
|
@lock_args_method ||= default_lock_args_method
|
105
105
|
end
|
106
106
|
|
107
107
|
# The global worker options defined in Sidekiq directly
|
108
108
|
def default_lock_args_method
|
109
|
-
|
110
|
-
|
109
|
+
default_job_options[LOCK_ARGS_METHOD] ||
|
110
|
+
default_job_options[UNIQUE_ARGS_METHOD]
|
111
111
|
end
|
112
112
|
|
113
113
|
#
|
@@ -116,8 +116,12 @@ module SidekiqUniqueJobs
|
|
116
116
|
#
|
117
117
|
# @return [Hash<String, Object>]
|
118
118
|
#
|
119
|
-
def
|
120
|
-
@
|
119
|
+
def default_job_options
|
120
|
+
@default_job_options ||= if Sidekiq.respond_to?(:default_job_options)
|
121
|
+
Sidekiq.default_job_options.stringify_keys
|
122
|
+
else
|
123
|
+
Sidekiq.default_worker_options.stringify_keys
|
124
|
+
end
|
121
125
|
end
|
122
126
|
end
|
123
127
|
end
|
@@ -13,9 +13,9 @@ module SidekiqUniqueJobs
|
|
13
13
|
# @return [Symbol] the type of lock
|
14
14
|
attr_reader :type
|
15
15
|
#
|
16
|
-
# @!attribute [r]
|
17
|
-
# @return [Symbol] the
|
18
|
-
attr_reader :
|
16
|
+
# @!attribute [r] job
|
17
|
+
# @return [Symbol] the job class
|
18
|
+
attr_reader :job
|
19
19
|
#
|
20
20
|
# @!attribute [r] limit
|
21
21
|
# @return [Integer] the number of simultaneous locks
|
@@ -58,7 +58,7 @@ module SidekiqUniqueJobs
|
|
58
58
|
|
59
59
|
def initialize(job_hash = {})
|
60
60
|
@type = job_hash[LOCK]&.to_sym
|
61
|
-
@
|
61
|
+
@job = SidekiqUniqueJobs.safe_constantize(job_hash[CLASS])
|
62
62
|
@limit = job_hash.fetch(LOCK_LIMIT, 1)&.to_i
|
63
63
|
@timeout = job_hash.fetch(LOCK_TIMEOUT, 0)&.to_i
|
64
64
|
@ttl = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION, nil) }.to_i
|
@@ -36,10 +36,10 @@ module SidekiqUniqueJobs
|
|
36
36
|
|
37
37
|
# @param [Hash] item a Sidekiq job hash
|
38
38
|
def initialize(item)
|
39
|
-
@item
|
40
|
-
@
|
41
|
-
@
|
42
|
-
|
39
|
+
@item = item
|
40
|
+
@lock_args = item[LOCK_ARGS] || item[UNIQUE_ARGS] # TODO: Deprecate UNIQUE_ARGS
|
41
|
+
@lock_prefix = item[LOCK_PREFIX] || item[UNIQUE_PREFIX] # TODO: Deprecate UNIQUE_PREFIX
|
42
|
+
self.job_class = item[CLASS]
|
43
43
|
end
|
44
44
|
|
45
45
|
# Memoized lock_digest
|
@@ -67,13 +67,13 @@ module SidekiqUniqueJobs
|
|
67
67
|
# Checks if we should disregard the queue when creating the unique digest
|
68
68
|
# @return [true, false]
|
69
69
|
def unique_across_queues?
|
70
|
-
item[UNIQUE_ACROSS_QUEUES] ||
|
70
|
+
item[UNIQUE_ACROSS_QUEUES] || job_options[UNIQUE_ACROSS_QUEUES]
|
71
71
|
end
|
72
72
|
|
73
73
|
# Checks if we should disregard the worker when creating the unique digest
|
74
74
|
# @return [true, false]
|
75
75
|
def unique_across_workers?
|
76
|
-
item[UNIQUE_ACROSS_WORKERS] ||
|
76
|
+
item[UNIQUE_ACROSS_WORKERS] || job_options[UNIQUE_ACROSS_WORKERS]
|
77
77
|
end
|
78
78
|
end
|
79
79
|
end
|
@@ -30,8 +30,8 @@ module SidekiqUniqueJobs
|
|
30
30
|
# @option item [String] :class the class of the sidekiq worker
|
31
31
|
# @option item [Float] :at the unix time the job is scheduled at
|
32
32
|
def initialize(item)
|
33
|
-
@item
|
34
|
-
|
33
|
+
@item = item
|
34
|
+
self.job_class = item[CLASS]
|
35
35
|
end
|
36
36
|
|
37
37
|
#
|
@@ -42,9 +42,9 @@ module SidekiqUniqueJobs
|
|
42
42
|
# @return [Integer, nil]
|
43
43
|
#
|
44
44
|
def calculate
|
45
|
-
timeout =
|
45
|
+
timeout = default_job_options[LOCK_TIMEOUT]
|
46
46
|
timeout = default_lock_timeout if default_lock_timeout
|
47
|
-
timeout =
|
47
|
+
timeout = job_options[LOCK_TIMEOUT] if job_options.key?(LOCK_TIMEOUT)
|
48
48
|
timeout
|
49
49
|
end
|
50
50
|
|
@@ -33,8 +33,8 @@ module SidekiqUniqueJobs
|
|
33
33
|
# @option item [String] :class the class of the sidekiq worker
|
34
34
|
# @option item [Float] :at the unix time the job is scheduled at
|
35
35
|
def initialize(item)
|
36
|
-
@item
|
37
|
-
|
36
|
+
@item = item
|
37
|
+
self.job_class = item[CLASS]
|
38
38
|
end
|
39
39
|
|
40
40
|
#
|
@@ -67,9 +67,9 @@ module SidekiqUniqueJobs
|
|
67
67
|
#
|
68
68
|
def calculate
|
69
69
|
ttl = item[LOCK_TTL]
|
70
|
-
ttl ||=
|
70
|
+
ttl ||= job_options[LOCK_TTL]
|
71
71
|
ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
|
72
|
-
ttl ||=
|
72
|
+
ttl ||= job_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
|
73
73
|
ttl ||= SidekiqUniqueJobs.config.lock_ttl
|
74
74
|
ttl && (ttl.to_i + time_until_scheduled)
|
75
75
|
end
|
@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
|
|
29
29
|
# @yieldparam [<type>] if <description>
|
30
30
|
# @yieldreturn [<type>] <describe what yield should return>
|
31
31
|
def call(worker_class, item, queue, redis_pool = nil)
|
32
|
-
@
|
33
|
-
@
|
34
|
-
@
|
35
|
-
|
32
|
+
@item = item
|
33
|
+
@queue = queue
|
34
|
+
@redis_pool = redis_pool
|
35
|
+
self.job_class = item[CLASS]
|
36
36
|
return yield if unique_disabled?
|
37
37
|
|
38
38
|
SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]
|
@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
|
|
14
14
|
# @param [Hash] item sidekiq job hash
|
15
15
|
def initialize(item, redis_pool = nil)
|
16
16
|
super(item, redis_pool)
|
17
|
-
|
17
|
+
self.job_class = item[CLASS]
|
18
18
|
end
|
19
19
|
|
20
20
|
# Create a new job from the current one.
|
21
21
|
# This will mess up sidekiq stats because a new job is created
|
22
22
|
def call
|
23
|
-
if
|
24
|
-
if
|
23
|
+
if sidekiq_job_class?
|
24
|
+
if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
|
25
25
|
reflect(:rescheduled, item)
|
26
26
|
else
|
27
27
|
reflect(:reschedule_failed, item)
|
@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
|
|
6
6
|
# Requires the following methods to be defined in the including class
|
7
7
|
# 1. item (required)
|
8
8
|
# 2. options (can be nil)
|
9
|
-
# 3.
|
9
|
+
# 3. job_class (required, can be anything)
|
10
10
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
11
11
|
module OptionsWithFallback
|
12
12
|
def self.included(base)
|
@@ -69,8 +69,8 @@ module SidekiqUniqueJobs
|
|
69
69
|
#
|
70
70
|
def options
|
71
71
|
@options ||= begin
|
72
|
-
opts =
|
73
|
-
opts.merge!(
|
72
|
+
opts = default_job_options.dup
|
73
|
+
opts.merge!(job_options) if sidekiq_job_class?
|
74
74
|
(opts || {}).stringify_keys
|
75
75
|
end
|
76
76
|
end
|
@@ -110,13 +110,7 @@ module SidekiqUniqueJobs
|
|
110
110
|
# @return [Hash]
|
111
111
|
#
|
112
112
|
def timer_task_options
|
113
|
-
|
114
|
-
|
115
|
-
if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
|
116
|
-
timer_task_options[:timeout_interval] = reaper_timeout
|
117
|
-
end
|
118
|
-
|
119
|
-
timer_task_options
|
113
|
+
{ run_now: true, execution_interval: reaper_interval }
|
120
114
|
end
|
121
115
|
|
122
116
|
#
|
@@ -133,13 +127,6 @@ module SidekiqUniqueJobs
|
|
133
127
|
SidekiqUniqueJobs.config.reaper_interval
|
134
128
|
end
|
135
129
|
|
136
|
-
#
|
137
|
-
# @see SidekiqUniqueJobs::Config#reaper_timeout
|
138
|
-
#
|
139
|
-
def reaper_timeout
|
140
|
-
SidekiqUniqueJobs.config.reaper_timeout
|
141
|
-
end
|
142
|
-
|
143
130
|
#
|
144
131
|
# A context to use for all log entries
|
145
132
|
#
|
@@ -11,6 +11,8 @@ module SidekiqUniqueJobs
|
|
11
11
|
#
|
12
12
|
# rubocop:disable Metrics/ClassLength
|
13
13
|
class RubyReaper < Reaper
|
14
|
+
include SidekiqUniqueJobs::Timing
|
15
|
+
|
14
16
|
#
|
15
17
|
# @return [String] the suffix for :RUN locks
|
16
18
|
RUN_SUFFIX = ":RUN"
|
@@ -30,6 +32,16 @@ module SidekiqUniqueJobs
|
|
30
32
|
# @return [Redis::SortedSet] the Sidekiq RetrySet
|
31
33
|
attr_reader :retried
|
32
34
|
|
35
|
+
#
|
36
|
+
# @!attribute [r] start_time
|
37
|
+
# @return [Integer] The timestamp this execution started represented as integer
|
38
|
+
attr_reader :start_time
|
39
|
+
|
40
|
+
#
|
41
|
+
# @!attribute [r] timeout_ms
|
42
|
+
# @return [Integer] The allowed ms before timeout
|
43
|
+
attr_reader :timeout_ms
|
44
|
+
|
33
45
|
#
|
34
46
|
# Initialize a new instance of DeleteOrphans
|
35
47
|
#
|
@@ -37,9 +49,11 @@ module SidekiqUniqueJobs
|
|
37
49
|
#
|
38
50
|
def initialize(conn)
|
39
51
|
super(conn)
|
40
|
-
@digests
|
41
|
-
@scheduled
|
42
|
-
@retried
|
52
|
+
@digests = SidekiqUniqueJobs::Digests.new
|
53
|
+
@scheduled = Redis::SortedSet.new(SCHEDULE)
|
54
|
+
@retried = Redis::SortedSet.new(RETRY)
|
55
|
+
@start_time = time_source.call
|
56
|
+
@timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
|
43
57
|
end
|
44
58
|
|
45
59
|
#
|
@@ -60,7 +74,7 @@ module SidekiqUniqueJobs
|
|
60
74
|
#
|
61
75
|
# @return [Array<String>] an array of orphaned digests
|
62
76
|
#
|
63
|
-
def orphans # rubocop:disable Metrics/MethodLength
|
77
|
+
def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
|
64
78
|
page = 0
|
65
79
|
per = reaper_count * 2
|
66
80
|
orphans = []
|
@@ -68,12 +82,14 @@ module SidekiqUniqueJobs
|
|
68
82
|
|
69
83
|
while results.size.positive?
|
70
84
|
results.each do |digest|
|
85
|
+
break if timeout?
|
71
86
|
next if belongs_to_job?(digest)
|
72
87
|
|
73
88
|
orphans << digest
|
74
89
|
break if orphans.size >= reaper_count
|
75
90
|
end
|
76
91
|
|
92
|
+
break if timeout?
|
77
93
|
break if orphans.size >= reaper_count
|
78
94
|
|
79
95
|
page += 1
|
@@ -83,6 +99,14 @@ module SidekiqUniqueJobs
|
|
83
99
|
orphans
|
84
100
|
end
|
85
101
|
|
102
|
+
def timeout?
|
103
|
+
elapsed_ms >= timeout_ms
|
104
|
+
end
|
105
|
+
|
106
|
+
def elapsed_ms
|
107
|
+
time_source.call - start_time
|
108
|
+
end
|
109
|
+
|
86
110
|
#
|
87
111
|
# Checks if the digest has a matching job.
|
88
112
|
# 1. It checks the scheduled set
|
@@ -5,50 +5,55 @@ module SidekiqUniqueJobs
|
|
5
5
|
#
|
6
6
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
7
|
module SidekiqWorkerMethods
|
8
|
+
#
|
9
|
+
# @!attribute [r] job_class
|
10
|
+
# @return [Sidekiq::Worker] The Sidekiq::Worker implementation
|
11
|
+
attr_reader :job_class
|
12
|
+
|
8
13
|
# Avoids duplicating worker_class.respond_to? in multiple places
|
9
14
|
# @return [true, false]
|
10
|
-
def
|
11
|
-
|
15
|
+
def job_method_defined?(method_sym)
|
16
|
+
job_class.respond_to?(method_sym)
|
12
17
|
end
|
13
18
|
|
14
19
|
# Wraps #get_sidekiq_options to always work with a hash
|
15
20
|
# @return [Hash] of the worker class sidekiq options
|
16
|
-
def
|
17
|
-
return {} unless
|
21
|
+
def job_options
|
22
|
+
return {} unless sidekiq_job_class?
|
18
23
|
|
19
|
-
|
24
|
+
job_class.get_sidekiq_options.deep_stringify_keys
|
20
25
|
end
|
21
26
|
|
22
27
|
# Tests that the
|
23
|
-
# @return [true] if
|
24
|
-
# @return [false] if
|
25
|
-
def
|
26
|
-
|
28
|
+
# @return [true] if job_class responds to get_sidekiq_options
|
29
|
+
# @return [false] if job_class does not respond to get_sidekiq_options
|
30
|
+
def sidekiq_job_class?
|
31
|
+
job_method_defined?(:get_sidekiq_options)
|
27
32
|
end
|
28
33
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
@
|
34
|
+
def job_class=(obj)
|
35
|
+
# this is what was originally passed in, it can be an instance or a class depending on sidekiq version
|
36
|
+
@original_job_class = obj
|
37
|
+
@job_class = job_class_constantize(obj)
|
33
38
|
end
|
34
39
|
|
35
40
|
# The hook to call after a successful unlock
|
36
41
|
# @return [Proc]
|
37
42
|
def after_unlock_hook # rubocop:disable Metrics/MethodLength
|
38
43
|
lambda do
|
39
|
-
if @
|
44
|
+
if @original_job_class.respond_to?(:after_unlock)
|
40
45
|
# instance method in sidekiq v6
|
41
|
-
if @
|
42
|
-
@
|
46
|
+
if @original_job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
|
47
|
+
@original_job_class.after_unlock(item)
|
43
48
|
else
|
44
|
-
@
|
49
|
+
@original_job_class.after_unlock
|
45
50
|
end
|
46
|
-
elsif
|
51
|
+
elsif job_class.respond_to?(:after_unlock)
|
47
52
|
# class method regardless of sidekiq version
|
48
|
-
if
|
49
|
-
|
53
|
+
if job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
|
54
|
+
job_class.after_unlock(item)
|
50
55
|
else
|
51
|
-
|
56
|
+
job_class.after_unlock
|
52
57
|
end
|
53
58
|
end
|
54
59
|
end
|
@@ -58,7 +63,7 @@ module SidekiqUniqueJobs
|
|
58
63
|
# failing back to the original argument when the constant can't be found
|
59
64
|
#
|
60
65
|
# @return [Sidekiq::Worker]
|
61
|
-
def
|
66
|
+
def job_class_constantize(klazz = @job_class)
|
62
67
|
SidekiqUniqueJobs.safe_constantize(klazz)
|
63
68
|
end
|
64
69
|
|
@@ -68,8 +73,12 @@ module SidekiqUniqueJobs
|
|
68
73
|
#
|
69
74
|
# @return [Hash<Symbol, Object>]
|
70
75
|
#
|
71
|
-
def
|
72
|
-
Sidekiq.
|
76
|
+
def default_job_options
|
77
|
+
if Sidekiq.respond_to?(:default_job_options)
|
78
|
+
Sidekiq.default_job_options
|
79
|
+
else
|
80
|
+
Sidekiq.default_worker_options
|
81
|
+
end
|
73
82
|
end
|
74
83
|
end
|
75
84
|
end
|
@@ -1,84 +1,299 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "concurrent/
|
4
|
-
|
3
|
+
require "concurrent/collection/copy_on_notify_observer_set"
|
4
|
+
require "concurrent/concern/dereferenceable"
|
5
|
+
require "concurrent/concern/observable"
|
6
|
+
require "concurrent/atomic/atomic_boolean"
|
7
|
+
require "concurrent/executor/executor_service"
|
8
|
+
require "concurrent/executor/ruby_executor_service"
|
9
|
+
require "concurrent/executor/safe_task_executor"
|
10
|
+
require "concurrent/scheduled_task"
|
5
11
|
|
6
12
|
module SidekiqUniqueJobs
|
7
|
-
#
|
13
|
+
# A very common concurrency pattern is to run a thread that performs a task at
|
14
|
+
# regular intervals. The thread that performs the task sleeps for the given
|
15
|
+
# interval then wakes up and performs the task. Lather, rinse, repeat... This
|
16
|
+
# pattern causes two problems. First, it is difficult to test the business
|
17
|
+
# logic of the task because the task itself is tightly coupled with the
|
18
|
+
# concurrency logic. Second, an exception raised while performing the task can
|
19
|
+
# cause the entire thread to abend. In a long-running application where the
|
20
|
+
# task thread is intended to run for days/weeks/years a crashed task thread
|
21
|
+
# can pose a significant problem. `TimerTask` alleviates both problems.
|
8
22
|
#
|
9
|
-
|
10
|
-
|
23
|
+
# When a `TimerTask` is launched it starts a thread for monitoring the
|
24
|
+
# execution interval. The `TimerTask` thread does not perform the task,
|
25
|
+
# however. Instead, the TimerTask launches the task on a separate thread.
|
26
|
+
# Should the task experience an unrecoverable crash only the task thread will
|
27
|
+
# crash. This makes the `TimerTask` very fault tolerant. Additionally, the
|
28
|
+
# `TimerTask` thread can respond to the success or failure of the task,
|
29
|
+
# performing logging or ancillary operations.
|
30
|
+
#
|
31
|
+
# One other advantage of `TimerTask` is that it forces the business logic to
|
32
|
+
# be completely decoupled from the concurrency logic. The business logic can
|
33
|
+
# be tested separately then passed to the `TimerTask` for scheduling and
|
34
|
+
# running.
|
35
|
+
#
|
36
|
+
# In some cases it may be necessary for a `TimerTask` to affect its own
|
37
|
+
# execution cycle. To facilitate this, a reference to the TimerTask instance
|
38
|
+
# is passed as an argument to the provided block every time the task is
|
39
|
+
# executed.
|
40
|
+
#
|
41
|
+
# The `TimerTask` class includes the `Dereferenceable` mixin module so the
|
42
|
+
# result of the last execution is always available via the `#value` method.
|
43
|
+
# Dereferencing options can be passed to the `TimerTask` during construction or
|
44
|
+
# at any later time using the `#set_deref_options` method.
|
45
|
+
#
|
46
|
+
# `TimerTask` supports notification through the Ruby standard library
|
47
|
+
# {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
|
48
|
+
# Observable} module. On execution the `TimerTask` will notify the observers
|
49
|
+
# with three arguments: time of execution, the result of the block (or nil on
|
50
|
+
# failure), and any raised exceptions (or nil on success).
|
51
|
+
#
|
52
|
+
# @!macro copy_options
|
53
|
+
#
|
54
|
+
# @example Basic usage
|
55
|
+
# task = Concurrent::TimerTask.new{ puts 'Boom!' }
|
56
|
+
# task.execute
|
57
|
+
#
|
58
|
+
# task.execution_interval #=> 60 (default)
|
59
|
+
#
|
60
|
+
# # wait 60 seconds...
|
61
|
+
# #=> 'Boom!'
|
62
|
+
#
|
63
|
+
# task.shutdown #=> true
|
64
|
+
#
|
65
|
+
# @example Configuring `:execution_interval`
|
66
|
+
# task = Concurrent::TimerTask.new(execution_interval: 5) do
|
67
|
+
# puts 'Boom!'
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# task.execution_interval #=> 5
|
71
|
+
#
|
72
|
+
# @example Immediate execution with `:run_now`
|
73
|
+
# task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
|
74
|
+
# task.execute
|
75
|
+
#
|
76
|
+
# #=> 'Boom!'
|
77
|
+
#
|
78
|
+
# @example Last `#value` and `Dereferenceable` mixin
|
79
|
+
# task = Concurrent::TimerTask.new(
|
80
|
+
# dup_on_deref: true,
|
81
|
+
# execution_interval: 5
|
82
|
+
# ){ Time.now }
|
83
|
+
#
|
84
|
+
# task.execute
|
85
|
+
# Time.now #=> 2013-11-07 18:06:50 -0500
|
86
|
+
# sleep(10)
|
87
|
+
# task.value #=> 2013-11-07 18:06:55 -0500
|
88
|
+
#
|
89
|
+
# @example Controlling execution from within the block
|
90
|
+
# timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
|
91
|
+
# task.execution_interval.times{ print 'Boom! ' }
|
92
|
+
# print "\n"
|
93
|
+
# task.execution_interval += 1
|
94
|
+
# if task.execution_interval > 5
|
95
|
+
# puts 'Stopping...'
|
96
|
+
# task.shutdown
|
97
|
+
# end
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
# timer_task.execute # blocking call - this task will stop itself
|
101
|
+
# #=> Boom!
|
102
|
+
# #=> Boom! Boom!
|
103
|
+
# #=> Boom! Boom! Boom!
|
104
|
+
# #=> Boom! Boom! Boom! Boom!
|
105
|
+
# #=> Boom! Boom! Boom! Boom! Boom!
|
106
|
+
# #=> Stopping...
|
107
|
+
#
|
108
|
+
# @example Observation
|
109
|
+
# class TaskObserver
|
110
|
+
# def update(time, result, ex)
|
111
|
+
# if result
|
112
|
+
# print "(#{time}) Execution successfully returned #{result}\n"
|
113
|
+
# else
|
114
|
+
# print "(#{time}) Execution failed with error #{ex}\n"
|
115
|
+
# end
|
116
|
+
# end
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
|
120
|
+
# task.add_observer(TaskObserver.new)
|
121
|
+
# task.execute
|
122
|
+
# sleep 4
|
123
|
+
#
|
124
|
+
# #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
|
125
|
+
# #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
|
126
|
+
# #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
|
127
|
+
# task.shutdown
|
128
|
+
#
|
129
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
|
130
|
+
# task.add_observer(TaskObserver.new)
|
131
|
+
# task.execute
|
132
|
+
#
|
133
|
+
# #=> (2013-10-13 19:07:25 -0400) Execution timed out
|
134
|
+
# #=> (2013-10-13 19:07:27 -0400) Execution timed out
|
135
|
+
# #=> (2013-10-13 19:07:29 -0400) Execution timed out
|
136
|
+
# task.shutdown
|
137
|
+
#
|
138
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
|
139
|
+
# task.add_observer(TaskObserver.new)
|
140
|
+
# task.execute
|
141
|
+
#
|
142
|
+
# #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
|
143
|
+
# #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
|
144
|
+
# #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
|
145
|
+
# task.shutdown
|
146
|
+
#
|
147
|
+
# @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
|
148
|
+
# @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
|
149
|
+
class TimerTask < Concurrent::RubyExecutorService
|
150
|
+
include Concurrent::Concern::Dereferenceable
|
151
|
+
include Concurrent::Concern::Observable
|
11
152
|
|
12
|
-
|
153
|
+
# Default `:execution_interval` in seconds.
|
154
|
+
EXECUTION_INTERVAL = 60
|
13
155
|
|
14
|
-
|
15
|
-
|
156
|
+
# Default `:timeout_interval` in seconds.
|
157
|
+
TIMEOUT_INTERVAL = 30
|
16
158
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
159
|
+
# Create a new TimerTask with the given task and configuration.
|
160
|
+
#
|
161
|
+
# @!macro timer_task_initialize
|
162
|
+
# @param [Hash] opts the options defining task execution.
|
163
|
+
# @option opts [Integer] :execution_interval number of seconds between
|
164
|
+
# task executions (default: EXECUTION_INTERVAL)
|
165
|
+
# @option opts [Boolean] :run_now Whether to run the task immediately
|
166
|
+
# upon instantiation or to wait until the first # execution_interval
|
167
|
+
# has passed (default: false)
|
168
|
+
#
|
169
|
+
# @!macro deref_options
|
170
|
+
#
|
171
|
+
# @raise ArgumentError when no block is given.
|
172
|
+
#
|
173
|
+
# @yield to the block after :execution_interval seconds have passed since
|
174
|
+
# the last yield
|
175
|
+
# @yieldparam task a reference to the `TimerTask` instance so that the
|
176
|
+
# block can control its own lifecycle. Necessary since `self` will
|
177
|
+
# refer to the execution context of the block rather than the running
|
178
|
+
# `TimerTask`.
|
179
|
+
#
|
180
|
+
# @return [TimerTask] the new `TimerTask`
|
181
|
+
def initialize(opts = {}, &task)
|
182
|
+
raise ArgumentError, "no block given" unless task
|
24
183
|
|
25
|
-
|
26
|
-
|
184
|
+
super
|
185
|
+
set_deref_options opts
|
186
|
+
end
|
27
187
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
188
|
+
# Is the executor running?
|
189
|
+
#
|
190
|
+
# @return [Boolean] `true` when running, `false` when shutting down or shutdown
|
191
|
+
def running?
|
192
|
+
@running.true?
|
193
|
+
end
|
33
194
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
@
|
52
|
-
ensure
|
53
|
-
@thread_completed.set
|
195
|
+
# Execute a previously created `TimerTask`.
|
196
|
+
#
|
197
|
+
# @return [TimerTask] a reference to `self`
|
198
|
+
#
|
199
|
+
# @example Instance and execute in separate steps
|
200
|
+
# task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
|
201
|
+
# task.running? #=> false
|
202
|
+
# task.execute
|
203
|
+
# task.running? #=> true
|
204
|
+
#
|
205
|
+
# @example Instance and execute in one line
|
206
|
+
# task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
|
207
|
+
# task.running? #=> true
|
208
|
+
def execute
|
209
|
+
synchronize do
|
210
|
+
if @running.false?
|
211
|
+
@running.make_true
|
212
|
+
schedule_next_task(@run_now ? 0 : @execution_interval)
|
54
213
|
end
|
214
|
+
end
|
215
|
+
self
|
216
|
+
end
|
55
217
|
|
56
|
-
|
218
|
+
# Create and execute a new `TimerTask`.
|
219
|
+
#
|
220
|
+
# @!macro timer_task_initialize
|
221
|
+
#
|
222
|
+
# @example
|
223
|
+
# task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
|
224
|
+
# task.running? #=> true
|
225
|
+
def self.execute(opts = {}, &task)
|
226
|
+
TimerTask.new(opts, &task).execute
|
227
|
+
end
|
57
228
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
229
|
+
# @!attribute [rw] execution_interval
|
230
|
+
# @return [Fixnum] Number of seconds after the task completes before the
|
231
|
+
# task is performed again.
|
232
|
+
def execution_interval
|
233
|
+
synchronize { @execution_interval }
|
234
|
+
end
|
235
|
+
|
236
|
+
# @!attribute [rw] execution_interval
|
237
|
+
# @return [Fixnum] Number of seconds after the task completes before the
|
238
|
+
# task is performed again.
|
239
|
+
def execution_interval=(value)
|
240
|
+
raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
|
241
|
+
|
242
|
+
synchronize { @execution_interval = value }
|
243
|
+
end
|
244
|
+
|
245
|
+
private :post, :<<
|
246
|
+
|
247
|
+
private
|
248
|
+
|
249
|
+
def ns_initialize(opts, &task)
|
250
|
+
set_deref_options(opts)
|
251
|
+
|
252
|
+
self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
|
253
|
+
if opts[:timeout] || opts[:timeout_interval]
|
254
|
+
warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
|
66
255
|
end
|
256
|
+
@run_now = opts[:now] || opts[:run_now]
|
257
|
+
@executor = Concurrent::SafeTaskExecutor.new(task)
|
258
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
259
|
+
@value = nil
|
260
|
+
|
261
|
+
self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
|
262
|
+
end
|
263
|
+
|
264
|
+
# @!visibility private
|
265
|
+
def ns_shutdown_execution
|
266
|
+
@running.make_false
|
267
|
+
super
|
268
|
+
end
|
67
269
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
270
|
+
# @!visibility private
|
271
|
+
def ns_kill_execution
|
272
|
+
@running.make_false
|
273
|
+
super
|
274
|
+
end
|
72
275
|
|
73
|
-
|
74
|
-
|
75
|
-
|
276
|
+
# @!visibility private
|
277
|
+
def schedule_next_task(interval = execution_interval)
|
278
|
+
exec_task = ->(completion) { execute_task(completion) }
|
279
|
+
Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
|
280
|
+
nil
|
281
|
+
end
|
76
282
|
|
77
|
-
|
283
|
+
# @!visibility private
|
284
|
+
def execute_task(completion)
|
285
|
+
return nil unless @running.true?
|
78
286
|
|
287
|
+
_success, value, reason = @executor.execute(self)
|
288
|
+
if completion.try?
|
289
|
+
self.value = value
|
79
290
|
schedule_next_task
|
80
|
-
|
291
|
+
time = Time.now
|
292
|
+
observers.notify_observers do
|
293
|
+
[time, self.value, reason]
|
294
|
+
end
|
81
295
|
end
|
296
|
+
nil
|
82
297
|
end
|
83
298
|
end
|
84
299
|
end
|
data/lib/tasks/changelog.rake
CHANGED
@@ -17,7 +17,7 @@ COMMIT_CHANGELOG_CMD = "git commit -a -m 'Update changelog'"
|
|
17
17
|
desc "Generate a Changelog"
|
18
18
|
task :changelog do
|
19
19
|
sh("git checkout main")
|
20
|
-
sh(*CHANGELOG_CMD.push(ENV
|
20
|
+
sh(*CHANGELOG_CMD.push(ENV.fetch("CHANGELOG_GITHUB_TOKEN", nil)))
|
21
21
|
sh(ADD_CHANGELOG_CMD)
|
22
22
|
sh(COMMIT_CHANGELOG_CMD)
|
23
23
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-unique-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.1.
|
4
|
+
version: 7.1.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mikael Henriksson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-04-
|
11
|
+
date: 2022-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: brpoplpush-redis_script
|