sidekiq-unique-jobs 7.0.10 → 7.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -2
- data/README.md +95 -28
- data/lib/sidekiq_unique_jobs.rb +4 -0
- data/lib/sidekiq_unique_jobs/config.rb +12 -0
- data/lib/sidekiq_unique_jobs/constants.rb +0 -1
- data/lib/sidekiq_unique_jobs/deprecation.rb +35 -0
- data/lib/sidekiq_unique_jobs/exceptions.rb +9 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +56 -51
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +31 -9
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +17 -5
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +15 -1
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +21 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +11 -6
- data/lib/sidekiq_unique_jobs/lock_config.rb +4 -4
- data/lib/sidekiq_unique_jobs/locksmith.rb +84 -80
- data/lib/sidekiq_unique_jobs/middleware/client.rb +8 -10
- data/lib/sidekiq_unique_jobs/middleware/server.rb +2 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +7 -3
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +0 -9
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +1 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +1 -1
- data/lib/sidekiq_unique_jobs/reflectable.rb +17 -0
- data/lib/sidekiq_unique_jobs/reflections.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/caller.rb +3 -1
- data/lib/sidekiq_unique_jobs/server.rb +1 -0
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +37 -1
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +1 -8
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- metadata +7 -3
@@ -6,7 +6,12 @@ module SidekiqUniqueJobs
|
|
6
6
|
#
|
7
7
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
8
|
class Client
|
9
|
+
# prepend "SidekiqUniqueJobs::Middleware"
|
10
|
+
# @!parse prepends SidekiqUniqueJobs::Middleware
|
9
11
|
prepend SidekiqUniqueJobs::Middleware
|
12
|
+
# includes "SidekiqUniqueJobs::Reflectable"
|
13
|
+
# @!parse include SidekiqUniqueJobs::Reflectable
|
14
|
+
include SidekiqUniqueJobs::Reflectable
|
10
15
|
|
11
16
|
# Calls this client middleware
|
12
17
|
# Used from Sidekiq.process_single
|
@@ -25,18 +30,11 @@ module SidekiqUniqueJobs
|
|
25
30
|
private
|
26
31
|
|
27
32
|
def lock
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
warn_about_duplicate
|
33
|
+
lock_instance.lock do |_locked_jid|
|
34
|
+
reflect(:locked, item)
|
35
|
+
return yield
|
32
36
|
end
|
33
37
|
end
|
34
|
-
|
35
|
-
def warn_about_duplicate
|
36
|
-
return unless log_duplicate?
|
37
|
-
|
38
|
-
log_warn "Already locked with another job_id (#{dump_json(item)})"
|
39
|
-
end
|
40
38
|
end
|
41
39
|
end
|
42
40
|
end
|
@@ -9,6 +9,7 @@ module SidekiqUniqueJobs
|
|
9
9
|
include SidekiqUniqueJobs::SidekiqWorkerMethods
|
10
10
|
include SidekiqUniqueJobs::Logging
|
11
11
|
include SidekiqUniqueJobs::JSON
|
12
|
+
include SidekiqUniqueJobs::Reflectable
|
12
13
|
|
13
14
|
# @param [Hash] item sidekiq job hash
|
14
15
|
def initialize(item, redis_pool = nil)
|
@@ -20,10 +21,13 @@ module SidekiqUniqueJobs
|
|
20
21
|
# This will mess up sidekiq stats because a new job is created
|
21
22
|
def call
|
22
23
|
if sidekiq_worker_class?
|
23
|
-
|
24
|
-
|
24
|
+
if worker_class.perform_in(5, *item[ARGS])
|
25
|
+
reflect(:rescheduled, item)
|
26
|
+
else
|
27
|
+
reflect(:reschedule_failed, item)
|
28
|
+
end
|
25
29
|
else
|
26
|
-
|
30
|
+
reflect(:unknown_sidekiq_worker, item)
|
27
31
|
end
|
28
32
|
end
|
29
33
|
end
|
@@ -29,15 +29,6 @@ module SidekiqUniqueJobs
|
|
29
29
|
!unique_enabled?
|
30
30
|
end
|
31
31
|
|
32
|
-
# Should duplicate payloads be logged?
|
33
|
-
#
|
34
|
-
#
|
35
|
-
# @return [true, false, nil]
|
36
|
-
#
|
37
|
-
def log_duplicate?
|
38
|
-
options[LOG_DUPLICATE] || item[LOG_DUPLICATE]
|
39
|
-
end
|
40
|
-
|
41
32
|
#
|
42
33
|
# A new lock for this Sidekiq Job
|
43
34
|
#
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Orphans
|
5
|
+
# Restarts orphan manager if it is considered dead
|
6
|
+
module ReaperResurrector
|
7
|
+
module_function
|
8
|
+
|
9
|
+
include SidekiqUniqueJobs::Connection
|
10
|
+
include SidekiqUniqueJobs::Logging
|
11
|
+
|
12
|
+
DRIFT_FACTOR = 0.1
|
13
|
+
REAPERS = [:ruby, :lua].freeze
|
14
|
+
|
15
|
+
#
|
16
|
+
# Starts reaper resurrector that watches orphans reaper
|
17
|
+
#
|
18
|
+
# @return [SidekiqUniqueJobs::TimerTask] the task that was started
|
19
|
+
#
|
20
|
+
def start
|
21
|
+
return if resurrector_disabled?
|
22
|
+
return if reaper_disabled?
|
23
|
+
|
24
|
+
with_logging_context do
|
25
|
+
run_task
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Runs reaper resurrector task
|
31
|
+
#
|
32
|
+
# @return [SidekiqUniqueJobs::TimerTask]
|
33
|
+
def run_task
|
34
|
+
log_info("Starting Reaper Resurrector")
|
35
|
+
task.execute
|
36
|
+
task
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# The task that runs the resurrector
|
41
|
+
#
|
42
|
+
# @return [SidekiqUniqueJobs::TimerTask]
|
43
|
+
def task
|
44
|
+
SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
|
45
|
+
with_logging_context do
|
46
|
+
restart_if_dead
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
#
|
52
|
+
# Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
|
53
|
+
#
|
54
|
+
def restart_if_dead
|
55
|
+
return if reaper_registered?
|
56
|
+
|
57
|
+
log_info("Reaper is considered dead. Starting new reaper instance")
|
58
|
+
orphans_manager.start
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Returns orphan manager
|
63
|
+
#
|
64
|
+
# @return [SidekiqUniqueJobs::Orphans::Manager]
|
65
|
+
def orphans_manager
|
66
|
+
SidekiqUniqueJobs::Orphans::Manager
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# Checks if resurrector is disabled
|
71
|
+
#
|
72
|
+
# @see resurrector_enabled?
|
73
|
+
#
|
74
|
+
# @return [true, false]
|
75
|
+
def resurrector_disabled?
|
76
|
+
!resurrector_enabled?
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# Checks if resurrector is enabled
|
81
|
+
#
|
82
|
+
# @return [true, false]
|
83
|
+
def resurrector_enabled?
|
84
|
+
SidekiqUniqueJobs.config.reaper_resurrector_enabled
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Checks if reaping is disabled
|
89
|
+
#
|
90
|
+
# @see reaper_enabled?
|
91
|
+
#
|
92
|
+
# @return [true, false]
|
93
|
+
#
|
94
|
+
def reaper_disabled?
|
95
|
+
!reaper_enabled?
|
96
|
+
end
|
97
|
+
|
98
|
+
#
|
99
|
+
# Checks if reaping is enabled
|
100
|
+
#
|
101
|
+
# @return [true, false]
|
102
|
+
#
|
103
|
+
def reaper_enabled?
|
104
|
+
REAPERS.include?(reaper)
|
105
|
+
end
|
106
|
+
|
107
|
+
#
|
108
|
+
# Checks if reaper is registered
|
109
|
+
#
|
110
|
+
# @return [true, false]
|
111
|
+
def reaper_registered?
|
112
|
+
redis do |conn|
|
113
|
+
conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
#
|
118
|
+
# @see SidekiqUniqueJobs::Config#reaper
|
119
|
+
#
|
120
|
+
def reaper
|
121
|
+
SidekiqUniqueJobs.config.reaper
|
122
|
+
end
|
123
|
+
|
124
|
+
#
|
125
|
+
# Arguments passed on to the timer task
|
126
|
+
#
|
127
|
+
#
|
128
|
+
# @return [Hash]
|
129
|
+
#
|
130
|
+
def timer_task_options
|
131
|
+
{ run_now: false,
|
132
|
+
execution_interval: reaper_resurrector_interval }
|
133
|
+
end
|
134
|
+
|
135
|
+
#
|
136
|
+
# A context to use for all log entries
|
137
|
+
#
|
138
|
+
#
|
139
|
+
# @return [Hash] when logger responds to `:with_context`
|
140
|
+
# @return [String] when logger does not responds to `:with_context`
|
141
|
+
#
|
142
|
+
def logging_context
|
143
|
+
if logger_context_hash?
|
144
|
+
{ "uniquejobs" => "reaper-resurrector" }
|
145
|
+
else
|
146
|
+
"uniquejobs=reaper-resurrector"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
#
|
151
|
+
# @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
|
152
|
+
#
|
153
|
+
def reaper_resurrector_interval
|
154
|
+
SidekiqUniqueJobs.config.reaper_resurrector_interval
|
155
|
+
end
|
156
|
+
|
157
|
+
def reaper_interval
|
158
|
+
SidekiqUniqueJobs.config.reaper_interval
|
159
|
+
end
|
160
|
+
|
161
|
+
def drift_reaper_interval
|
162
|
+
reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
|
163
|
+
end
|
164
|
+
|
165
|
+
def current_timestamp
|
166
|
+
Time.now.to_i
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
#
|
5
|
+
# Module Reflectable provides a method to notify subscribers
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
#
|
9
|
+
module Reflectable
|
10
|
+
def reflect(name, *args)
|
11
|
+
SidekiqUniqueJobs.reflections.dispatch(name, *args)
|
12
|
+
nil
|
13
|
+
rescue UniqueJobsError => ex
|
14
|
+
SidekiqUniqueJobs.logger.error(ex)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
#
|
5
|
+
# Class NotificationCollection provides a collection with known notifications
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
#
|
9
|
+
class Reflections
|
10
|
+
#
|
11
|
+
# @return [Array<Symbol>] list of notifications
|
12
|
+
REFLECTIONS = [
|
13
|
+
:after_unlock_callback_failed,
|
14
|
+
:debug,
|
15
|
+
:duplicate,
|
16
|
+
:error,
|
17
|
+
:execution_failed,
|
18
|
+
:lock_failed,
|
19
|
+
:locked,
|
20
|
+
:reschedule_failed,
|
21
|
+
:rescheduled,
|
22
|
+
:timeout,
|
23
|
+
:unknown_sidekiq_worker,
|
24
|
+
:unlock_failed,
|
25
|
+
:unlocked,
|
26
|
+
].freeze
|
27
|
+
|
28
|
+
#
|
29
|
+
# @return [Hash<Symbol, Array<Symbol, String>>] a hash with deprecated notifications
|
30
|
+
DEPRECATIONS = {}.freeze
|
31
|
+
|
32
|
+
REFLECTIONS.each do |reflection|
|
33
|
+
class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
|
34
|
+
def #{reflection}(*args, &block) # def unlock_failed(*args, &block)
|
35
|
+
raise NoBlockGiven, "block required" unless block_given? # raise NoBlockGiven, "block required" unless block_given?
|
36
|
+
@reflections[:#{reflection}] = block # @notifications[:unlock_failed] = block
|
37
|
+
end # end
|
38
|
+
RUBY
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize
|
42
|
+
@reflections = {}
|
43
|
+
end
|
44
|
+
|
45
|
+
def dispatch(reflection, *args)
|
46
|
+
if (block = @reflections[reflection])
|
47
|
+
block.call(*args)
|
48
|
+
|
49
|
+
if DEPRECATIONS.key?(reflection)
|
50
|
+
replacement, removal_version = DEPRECATIONS[reflection]
|
51
|
+
SidekiqUniqueJobs::Deprecation.warn(
|
52
|
+
"#{reflection} is deprecated and will be removed in version #{removal_version}. Use #{replacement} instead.",
|
53
|
+
)
|
54
|
+
end
|
55
|
+
elsif misconfigured?(reflection)
|
56
|
+
raise NoSuchNotificationError, reflection
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def configured?(reflection)
|
61
|
+
REFLECTIONS.include?(reflection)
|
62
|
+
end
|
63
|
+
|
64
|
+
def misconfigured?(reflection)
|
65
|
+
!configured?(reflection)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -45,7 +45,9 @@ module SidekiqUniqueJobs
|
|
45
45
|
pool = defined?(redis_pool) ? redis_pool : nil
|
46
46
|
|
47
47
|
redis(pool) do |new_conn|
|
48
|
-
do_call(file_name, new_conn, keys, argv)
|
48
|
+
result = do_call(file_name, new_conn, keys, argv)
|
49
|
+
yield result if block_given?
|
50
|
+
result
|
49
51
|
end
|
50
52
|
end
|
51
53
|
|
@@ -4,7 +4,7 @@
|
|
4
4
|
# Contains configuration and utility methods that belongs top level
|
5
5
|
#
|
6
6
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
|
-
module SidekiqUniqueJobs
|
7
|
+
module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
|
8
8
|
include SidekiqUniqueJobs::Connection
|
9
9
|
extend SidekiqUniqueJobs::JSON
|
10
10
|
|
@@ -255,4 +255,40 @@ module SidekiqUniqueJobs
|
|
255
255
|
constant.const_get(name, false)
|
256
256
|
end
|
257
257
|
end
|
258
|
+
|
259
|
+
# Attempt to constantize a string worker_class argument, always
|
260
|
+
# failing back to the original argument when the constant can't be found
|
261
|
+
#
|
262
|
+
# @return [Sidekiq::Worker, String]
|
263
|
+
def safe_constantize(str)
|
264
|
+
constantize(str)
|
265
|
+
rescue NameError => ex
|
266
|
+
case ex.message
|
267
|
+
when /uninitialized constant/
|
268
|
+
str
|
269
|
+
else
|
270
|
+
raise
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
#
|
275
|
+
# Collection with notifications
|
276
|
+
#
|
277
|
+
#
|
278
|
+
# @return [Reflections]
|
279
|
+
#
|
280
|
+
def reflections
|
281
|
+
@reflections ||= Reflections.new
|
282
|
+
end
|
283
|
+
|
284
|
+
#
|
285
|
+
# Yields notification stack for sidekiq unique jobs to configure notifications
|
286
|
+
#
|
287
|
+
#
|
288
|
+
# @return [void] <description>
|
289
|
+
#
|
290
|
+
# @yieldparam [Reflections] x used to configure notifications
|
291
|
+
def reflect
|
292
|
+
yield reflections if block_given?
|
293
|
+
end
|
258
294
|
end
|
@@ -49,14 +49,7 @@ module SidekiqUniqueJobs
|
|
49
49
|
#
|
50
50
|
# @return [Sidekiq::Worker]
|
51
51
|
def worker_class_constantize(klazz = @worker_class)
|
52
|
-
SidekiqUniqueJobs.
|
53
|
-
rescue NameError => ex
|
54
|
-
case ex.message
|
55
|
-
when /uninitialized constant/
|
56
|
-
klazz
|
57
|
-
else
|
58
|
-
raise
|
59
|
-
end
|
52
|
+
SidekiqUniqueJobs.safe_constantize(klazz)
|
60
53
|
end
|
61
54
|
|
62
55
|
#
|