sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,242 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent/version"
4
+
5
+ module SidekiqUniqueJobs
6
+ module Orphans
7
+ #
8
+ # Manages the orphan reaper
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ module Manager
13
+ module_function
14
+
15
+ #
16
+ # @return [Float] the amount to add to the reaper interval
17
+ DRIFT_FACTOR = 0.02
18
+ #
19
+ # @return [Symbol] allowed reapers (:ruby or :lua)
20
+ REAPERS = [:ruby, :lua].freeze
21
+
22
+ # includes "SidekiqUniqueJobs::Connection"
23
+ # @!parse include SidekiqUniqueJobs::Connection
24
+ include SidekiqUniqueJobs::Connection
25
+ # includes "SidekiqUniqueJobs::Logging"
26
+ # @!parse include SidekiqUniqueJobs::Logging
27
+ include SidekiqUniqueJobs::Logging
28
+
29
+ #
30
+ # Starts a separate thread that periodically reaps orphans
31
+ #
32
+ #
33
+ # @return [SidekiqUniqueJobs::TimerTask] the task that was started
34
+ #
35
+ def start(test_task = nil) # rubocop:disable
36
+ return if disabled?
37
+ return if registered?
38
+
39
+ self.task = test_task || default_task
40
+
41
+ with_logging_context do
42
+ if register_reaper_process
43
+ log_info("Starting Reaper")
44
+
45
+ task.add_observer(Observer.new)
46
+ task.execute
47
+ task
48
+ end
49
+ end
50
+ end
51
+
52
+ #
53
+ # Stops the thread that reaps orphans
54
+ #
55
+ #
56
+ # @return [Boolean]
57
+ #
58
+ def stop
59
+ return if disabled?
60
+ return if unregistered?
61
+
62
+ with_logging_context do
63
+ log_info("Stopping Reaper")
64
+ unregister_reaper_process
65
+ task.shutdown
66
+ end
67
+ end
68
+
69
+ #
70
+ # The task that runs the reaper
71
+ #
72
+ #
73
+ # @return [<type>] <description>
74
+ #
75
+ def task
76
+ @task ||= default_task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
77
+ end
78
+
79
+ #
80
+ # A properly configured timer task
81
+ #
82
+ #
83
+ # @return [SidekiqUniqueJobs::TimerTask]
84
+ #
85
+ def default_task
86
+ SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
87
+ with_logging_context do
88
+ redis do |conn|
89
+ refresh_reaper_mutex
90
+ Orphans::Reaper.call(conn)
91
+ end
92
+ end
93
+ end
94
+ end
95
+
96
+ #
97
+ # Store a task to use for scheduled execution
98
+ #
99
+ # @param [SidekiqUniqueJobs::TimerTask] task the task to use
100
+ #
101
+ # @return [void]
102
+ #
103
+ def task=(task)
104
+ @task = task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
105
+ end
106
+
107
+ #
108
+ # Arguments passed on to the timer task
109
+ #
110
+ #
111
+ # @return [Hash]
112
+ #
113
+ def timer_task_options
114
+ { run_now: true, execution_interval: reaper_interval }
115
+ end
116
+
117
+ #
118
+ # @see SidekiqUniqueJobs::Config#reaper
119
+ #
120
+ def reaper
121
+ SidekiqUniqueJobs.config.reaper
122
+ end
123
+
124
+ #
125
+ # @see SidekiqUniqueJobs::Config#reaper_interval
126
+ #
127
+ def reaper_interval
128
+ SidekiqUniqueJobs.config.reaper_interval
129
+ end
130
+
131
+ #
132
+ # A context to use for all log entries
133
+ #
134
+ #
135
+ # @return [Hash] when logger responds to `:with_context`
136
+ # @return [String] when logger does not responds to `:with_context`
137
+ #
138
+ def logging_context
139
+ if logger_context_hash?
140
+ { "uniquejobs" => "reaper" }
141
+ else
142
+ "uniquejobs=orphan-reaper"
143
+ end
144
+ end
145
+
146
+ #
147
+ # Checks if a reaper is registered
148
+ #
149
+ #
150
+ # @return [true, false]
151
+ #
152
+ def registered?
153
+ redis do |conn|
154
+ conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
155
+ end
156
+ end
157
+
158
+ #
159
+ # Checks if that reapers are not registerd
160
+ #
161
+ # @see registered?
162
+ #
163
+ # @return [true, false]
164
+ #
165
+ def unregistered?
166
+ !registered?
167
+ end
168
+
169
+ #
170
+ # Checks if reaping is disabled
171
+ #
172
+ # @see enabled?
173
+ #
174
+ # @return [true, false]
175
+ #
176
+ def disabled?
177
+ !enabled?
178
+ end
179
+
180
+ #
181
+ # Checks if reaping is enabled
182
+ #
183
+ # @return [true, false]
184
+ #
185
+ def enabled?
186
+ REAPERS.include?(reaper)
187
+ end
188
+
189
+ #
190
+ # Writes a mutex key to redis
191
+ #
192
+ #
193
+ # @return [void]
194
+ #
195
+ def register_reaper_process
196
+ redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, "nx", "ex", drift_reaper_interval) }
197
+ end
198
+
199
+ #
200
+ # Updates mutex key
201
+ #
202
+ #
203
+ # @return [void]
204
+ #
205
+ def refresh_reaper_mutex
206
+ redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, "ex", drift_reaper_interval) }
207
+ end
208
+
209
+ #
210
+ # Removes mutex key from redis
211
+ #
212
+ #
213
+ # @return [void]
214
+ #
215
+ def unregister_reaper_process
216
+ redis { |conn| conn.del(UNIQUE_REAPER) }
217
+ end
218
+
219
+ #
220
+ # Reaper interval with a little drift
221
+ # Redis isn't exact enough so to give a little buffer,
222
+ # we add a tiny value to the reaper interval.
223
+ #
224
+ #
225
+ # @return [Integer] <description>
226
+ #
227
+ def drift_reaper_interval
228
+ reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
229
+ end
230
+
231
+ #
232
+ # Current time (as integer value)
233
+ #
234
+ #
235
+ # @return [Integer]
236
+ #
237
+ def current_timestamp
238
+ Time.now.to_i
239
+ end
240
+ end
241
+ end
242
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ #
6
+ # Class DeleteOrphans provides deletion of orphaned digests
7
+ #
8
+ # @note this is a much slower version of the lua script but does not crash redis
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ class NullReaper < Reaper
13
+ #
14
+ # Delete orphaned digests
15
+ #
16
+ #
17
+ # @return [Integer] the number of reaped locks
18
+ #
19
+ def call
20
+ # NO OP
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class DeleteOrphans provides deletion of orphaned digests
6
+ #
7
+ # @note this is a much slower version of the lua script but does not crash redis
8
+ #
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
+ #
11
+ module Orphans
12
+ #
13
+ # Observes the Orphan::Manager and provides information about each execution
14
+ #
15
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
16
+ #
17
+ class Observer
18
+ include SidekiqUniqueJobs::Logging
19
+
20
+ #
21
+ # Runs every time the {Manager} executes the TimerTask
22
+ # used for logging information about the reaping
23
+ #
24
+ # @param [Time] time the time of the execution
25
+ # @param [Object] result the result of the execution
26
+ # @param [Exception] ex any error raised from the TimerTask
27
+ #
28
+ # @return [<type>] <description>
29
+ #
30
+ def update(time, result, ex)
31
+ if result
32
+ log_info("(#{time}) Execution successfully returned #{result}")
33
+ elsif ex.is_a?(Concurrent::TimeoutError)
34
+ log_warn("(#{time}) Execution timed out")
35
+ else
36
+ log_info("(#{time}) Cleanup failed with error #{ex.message}")
37
+ log_error(ex)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ #
6
+ # Class DeleteOrphans provides deletion of orphaned digests
7
+ #
8
+ # @note this is a much slower version of the lua script but does not crash redis
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ class Reaper
13
+ include SidekiqUniqueJobs::Connection
14
+ include SidekiqUniqueJobs::Script::Caller
15
+ include SidekiqUniqueJobs::Logging
16
+ include SidekiqUniqueJobs::JSON
17
+
18
+ require_relative "lua_reaper"
19
+ require_relative "ruby_reaper"
20
+ require_relative "null_reaper"
21
+
22
+ #
23
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Orphans::Reaper] the current implementation of reapers
24
+ REAPERS = {
25
+ lua: SidekiqUniqueJobs::Orphans::RubyReaper,
26
+ ruby: SidekiqUniqueJobs::Orphans::RubyReaper,
27
+ none: SidekiqUniqueJobs::Orphans::NullReaper,
28
+ nil => SidekiqUniqueJobs::Orphans::NullReaper,
29
+ false => SidekiqUniqueJobs::Orphans::NullReaper,
30
+ true => SidekiqUniqueJobs::Orphans::RubyReaper,
31
+ }.freeze
32
+
33
+ #
34
+ # Execute deletion of orphaned digests
35
+ #
36
+ # @param [Redis] conn nil a connection to redis
37
+ #
38
+ # @return [void]
39
+ #
40
+ def self.call(conn = nil)
41
+ return new(conn).call if conn
42
+
43
+ redis { |rcon| new(rcon).call }
44
+ end
45
+
46
+ #
47
+ # @!attribute [r] conn
48
+ # @return [Redis] a redis connection
49
+ attr_reader :conn
50
+
51
+ #
52
+ # Initialize a new instance of DeleteOrphans
53
+ #
54
+ # @param [Redis] conn a connection to redis
55
+ #
56
+ def initialize(conn)
57
+ @conn = conn
58
+ end
59
+
60
+ #
61
+ # Convenient access to the global configuration
62
+ #
63
+ #
64
+ # @return [SidekiqUniqueJobs::Config]
65
+ #
66
+ def config
67
+ SidekiqUniqueJobs.config
68
+ end
69
+
70
+ #
71
+ # The reaper that was configured
72
+ #
73
+ #
74
+ # @return [Symbol]
75
+ #
76
+ def reaper
77
+ config.reaper
78
+ end
79
+
80
+ #
81
+ # The configured timeout for the reaper
82
+ #
83
+ #
84
+ # @return [Integer] timeout in seconds
85
+ #
86
+ def reaper_timeout
87
+ config.reaper_timeout
88
+ end
89
+
90
+ #
91
+ # The number of locks to reap at a time
92
+ #
93
+ #
94
+ # @return [Integer]
95
+ #
96
+ def reaper_count
97
+ config.reaper_count
98
+ end
99
+
100
+ #
101
+ # Delete orphaned digests
102
+ #
103
+ #
104
+ # @return [Integer] the number of reaped locks
105
+ #
106
+ def call
107
+ if (implementation = REAPERS[reaper])
108
+ implementation.new(conn).call
109
+ else
110
+ log_fatal(":#{reaper} is invalid for `SidekiqUnqiueJobs.config.reaper`")
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,170 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ # Restarts orphan manager if it is considered dead
6
+ module ReaperResurrector
7
+ module_function
8
+
9
+ include SidekiqUniqueJobs::Connection
10
+ include SidekiqUniqueJobs::Logging
11
+
12
+ DRIFT_FACTOR = 0.1
13
+ REAPERS = [:ruby, :lua].freeze
14
+
15
+ #
16
+ # Starts reaper resurrector that watches orphans reaper
17
+ #
18
+ # @return [SidekiqUniqueJobs::TimerTask] the task that was started
19
+ #
20
+ def start
21
+ return if resurrector_disabled?
22
+ return if reaper_disabled?
23
+
24
+ with_logging_context do
25
+ run_task
26
+ end
27
+ end
28
+
29
+ #
30
+ # Runs reaper resurrector task
31
+ #
32
+ # @return [SidekiqUniqueJobs::TimerTask]
33
+ def run_task
34
+ log_info("Starting Reaper Resurrector")
35
+ task.execute
36
+ task
37
+ end
38
+
39
+ #
40
+ # The task that runs the resurrector
41
+ #
42
+ # @return [SidekiqUniqueJobs::TimerTask]
43
+ def task
44
+ SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
45
+ with_logging_context do
46
+ restart_if_dead
47
+ end
48
+ end
49
+ end
50
+
51
+ #
52
+ # Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
53
+ #
54
+ def restart_if_dead
55
+ return if reaper_registered?
56
+
57
+ log_info("Reaper is considered dead. Starting new reaper instance")
58
+ orphans_manager.start
59
+ end
60
+
61
+ #
62
+ # Returns orphan manager
63
+ #
64
+ # @return [SidekiqUniqueJobs::Orphans::Manager]
65
+ def orphans_manager
66
+ SidekiqUniqueJobs::Orphans::Manager
67
+ end
68
+
69
+ #
70
+ # Checks if resurrector is disabled
71
+ #
72
+ # @see resurrector_enabled?
73
+ #
74
+ # @return [true, false]
75
+ def resurrector_disabled?
76
+ !resurrector_enabled?
77
+ end
78
+
79
+ #
80
+ # Checks if resurrector is enabled
81
+ #
82
+ # @return [true, false]
83
+ def resurrector_enabled?
84
+ SidekiqUniqueJobs.config.reaper_resurrector_enabled
85
+ end
86
+
87
+ #
88
+ # Checks if reaping is disabled
89
+ #
90
+ # @see reaper_enabled?
91
+ #
92
+ # @return [true, false]
93
+ #
94
+ def reaper_disabled?
95
+ !reaper_enabled?
96
+ end
97
+
98
+ #
99
+ # Checks if reaping is enabled
100
+ #
101
+ # @return [true, false]
102
+ #
103
+ def reaper_enabled?
104
+ REAPERS.include?(reaper)
105
+ end
106
+
107
+ #
108
+ # Checks if reaper is registered
109
+ #
110
+ # @return [true, false]
111
+ def reaper_registered?
112
+ redis do |conn|
113
+ conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
114
+ end
115
+ end
116
+
117
+ #
118
+ # @see SidekiqUniqueJobs::Config#reaper
119
+ #
120
+ def reaper
121
+ SidekiqUniqueJobs.config.reaper
122
+ end
123
+
124
+ #
125
+ # Arguments passed on to the timer task
126
+ #
127
+ #
128
+ # @return [Hash]
129
+ #
130
+ def timer_task_options
131
+ { run_now: false,
132
+ execution_interval: reaper_resurrector_interval }
133
+ end
134
+
135
+ #
136
+ # A context to use for all log entries
137
+ #
138
+ #
139
+ # @return [Hash] when logger responds to `:with_context`
140
+ # @return [String] when logger does not responds to `:with_context`
141
+ #
142
+ def logging_context
143
+ if logger_context_hash?
144
+ { "uniquejobs" => "reaper-resurrector" }
145
+ else
146
+ "uniquejobs=reaper-resurrector"
147
+ end
148
+ end
149
+
150
+ #
151
+ # @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
152
+ #
153
+ def reaper_resurrector_interval
154
+ SidekiqUniqueJobs.config.reaper_resurrector_interval
155
+ end
156
+
157
+ def reaper_interval
158
+ SidekiqUniqueJobs.config.reaper_interval
159
+ end
160
+
161
+ def drift_reaper_interval
162
+ reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
163
+ end
164
+
165
+ def current_timestamp
166
+ Time.now.to_i
167
+ end
168
+ end
169
+ end
170
+ end