sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,299 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent/collection/copy_on_notify_observer_set"
4
+ require "concurrent/concern/dereferenceable"
5
+ require "concurrent/concern/observable"
6
+ require "concurrent/atomic/atomic_boolean"
7
+ require "concurrent/executor/executor_service"
8
+ require "concurrent/executor/ruby_executor_service"
9
+ require "concurrent/executor/safe_task_executor"
10
+ require "concurrent/scheduled_task"
11
+
12
+ module SidekiqUniqueJobs
13
+ # A very common concurrency pattern is to run a thread that performs a task at
14
+ # regular intervals. The thread that performs the task sleeps for the given
15
+ # interval then wakes up and performs the task. Lather, rinse, repeat... This
16
+ # pattern causes two problems. First, it is difficult to test the business
17
+ # logic of the task because the task itself is tightly coupled with the
18
+ # concurrency logic. Second, an exception raised while performing the task can
19
+ # cause the entire thread to abend. In a long-running application where the
20
+ # task thread is intended to run for days/weeks/years a crashed task thread
21
+ # can pose a significant problem. `TimerTask` alleviates both problems.
22
+ #
23
+ # When a `TimerTask` is launched it starts a thread for monitoring the
24
+ # execution interval. The `TimerTask` thread does not perform the task,
25
+ # however. Instead, the TimerTask launches the task on a separate thread.
26
+ # Should the task experience an unrecoverable crash only the task thread will
27
+ # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
28
+ # `TimerTask` thread can respond to the success or failure of the task,
29
+ # performing logging or ancillary operations.
30
+ #
31
+ # One other advantage of `TimerTask` is that it forces the business logic to
32
+ # be completely decoupled from the concurrency logic. The business logic can
33
+ # be tested separately then passed to the `TimerTask` for scheduling and
34
+ # running.
35
+ #
36
+ # In some cases it may be necessary for a `TimerTask` to affect its own
37
+ # execution cycle. To facilitate this, a reference to the TimerTask instance
38
+ # is passed as an argument to the provided block every time the task is
39
+ # executed.
40
+ #
41
+ # The `TimerTask` class includes the `Dereferenceable` mixin module so the
42
+ # result of the last execution is always available via the `#value` method.
43
+ # Dereferencing options can be passed to the `TimerTask` during construction or
44
+ # at any later time using the `#set_deref_options` method.
45
+ #
46
+ # `TimerTask` supports notification through the Ruby standard library
47
+ # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
48
+ # Observable} module. On execution the `TimerTask` will notify the observers
49
+ # with three arguments: time of execution, the result of the block (or nil on
50
+ # failure), and any raised exceptions (or nil on success).
51
+ #
52
+ # @!macro copy_options
53
+ #
54
+ # @example Basic usage
55
+ # task = Concurrent::TimerTask.new{ puts 'Boom!' }
56
+ # task.execute
57
+ #
58
+ # task.execution_interval #=> 60 (default)
59
+ #
60
+ # # wait 60 seconds...
61
+ # #=> 'Boom!'
62
+ #
63
+ # task.shutdown #=> true
64
+ #
65
+ # @example Configuring `:execution_interval`
66
+ # task = Concurrent::TimerTask.new(execution_interval: 5) do
67
+ # puts 'Boom!'
68
+ # end
69
+ #
70
+ # task.execution_interval #=> 5
71
+ #
72
+ # @example Immediate execution with `:run_now`
73
+ # task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
74
+ # task.execute
75
+ #
76
+ # #=> 'Boom!'
77
+ #
78
+ # @example Last `#value` and `Dereferenceable` mixin
79
+ # task = Concurrent::TimerTask.new(
80
+ # dup_on_deref: true,
81
+ # execution_interval: 5
82
+ # ){ Time.now }
83
+ #
84
+ # task.execute
85
+ # Time.now #=> 2013-11-07 18:06:50 -0500
86
+ # sleep(10)
87
+ # task.value #=> 2013-11-07 18:06:55 -0500
88
+ #
89
+ # @example Controlling execution from within the block
90
+ # timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
91
+ # task.execution_interval.times{ print 'Boom! ' }
92
+ # print "\n"
93
+ # task.execution_interval += 1
94
+ # if task.execution_interval > 5
95
+ # puts 'Stopping...'
96
+ # task.shutdown
97
+ # end
98
+ # end
99
+ #
100
+ # timer_task.execute # blocking call - this task will stop itself
101
+ # #=> Boom!
102
+ # #=> Boom! Boom!
103
+ # #=> Boom! Boom! Boom!
104
+ # #=> Boom! Boom! Boom! Boom!
105
+ # #=> Boom! Boom! Boom! Boom! Boom!
106
+ # #=> Stopping...
107
+ #
108
+ # @example Observation
109
+ # class TaskObserver
110
+ # def update(time, result, ex)
111
+ # if result
112
+ # print "(#{time}) Execution successfully returned #{result}\n"
113
+ # else
114
+ # print "(#{time}) Execution failed with error #{ex}\n"
115
+ # end
116
+ # end
117
+ # end
118
+ #
119
+ # task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
120
+ # task.add_observer(TaskObserver.new)
121
+ # task.execute
122
+ # sleep 4
123
+ #
124
+ # #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
125
+ # #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
126
+ # #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
127
+ # task.shutdown
128
+ #
129
+ # task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
130
+ # task.add_observer(TaskObserver.new)
131
+ # task.execute
132
+ #
133
+ # #=> (2013-10-13 19:07:25 -0400) Execution timed out
134
+ # #=> (2013-10-13 19:07:27 -0400) Execution timed out
135
+ # #=> (2013-10-13 19:07:29 -0400) Execution timed out
136
+ # task.shutdown
137
+ #
138
+ # task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
139
+ # task.add_observer(TaskObserver.new)
140
+ # task.execute
141
+ #
142
+ # #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
143
+ # #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
144
+ # #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
145
+ # task.shutdown
146
+ #
147
+ # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
148
+ # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
149
+ class TimerTask < Concurrent::RubyExecutorService
150
+ include Concurrent::Concern::Dereferenceable
151
+ include Concurrent::Concern::Observable
152
+
153
+ # Default `:execution_interval` in seconds.
154
+ EXECUTION_INTERVAL = 60
155
+
156
+ # Default `:timeout_interval` in seconds.
157
+ TIMEOUT_INTERVAL = 30
158
+
159
+ # Create a new TimerTask with the given task and configuration.
160
+ #
161
+ # @!macro timer_task_initialize
162
+ # @param [Hash] opts the options defining task execution.
163
+ # @option opts [Integer] :execution_interval number of seconds between
164
+ # task executions (default: EXECUTION_INTERVAL)
165
+ # @option opts [Boolean] :run_now Whether to run the task immediately
166
+ # upon instantiation or to wait until the first # execution_interval
167
+ # has passed (default: false)
168
+ #
169
+ # @!macro deref_options
170
+ #
171
+ # @raise ArgumentError when no block is given.
172
+ #
173
+ # @yield to the block after :execution_interval seconds have passed since
174
+ # the last yield
175
+ # @yieldparam task a reference to the `TimerTask` instance so that the
176
+ # block can control its own lifecycle. Necessary since `self` will
177
+ # refer to the execution context of the block rather than the running
178
+ # `TimerTask`.
179
+ #
180
+ # @return [TimerTask] the new `TimerTask`
181
+ def initialize(opts = {}, &task)
182
+ raise ArgumentError, "no block given" unless task
183
+
184
+ super
185
+ set_deref_options opts
186
+ end
187
+
188
+ # Is the executor running?
189
+ #
190
+ # @return [Boolean] `true` when running, `false` when shutting down or shutdown
191
+ def running?
192
+ @running.true?
193
+ end
194
+
195
+ # Execute a previously created `TimerTask`.
196
+ #
197
+ # @return [TimerTask] a reference to `self`
198
+ #
199
+ # @example Instance and execute in separate steps
200
+ # task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
201
+ # task.running? #=> false
202
+ # task.execute
203
+ # task.running? #=> true
204
+ #
205
+ # @example Instance and execute in one line
206
+ # task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
207
+ # task.running? #=> true
208
+ def execute
209
+ synchronize do
210
+ if @running.false?
211
+ @running.make_true
212
+ schedule_next_task(@run_now ? 0 : @execution_interval)
213
+ end
214
+ end
215
+ self
216
+ end
217
+
218
+ # Create and execute a new `TimerTask`.
219
+ #
220
+ # @!macro timer_task_initialize
221
+ #
222
+ # @example
223
+ # task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
224
+ # task.running? #=> true
225
+ def self.execute(opts = {}, &task)
226
+ TimerTask.new(opts, &task).execute
227
+ end
228
+
229
+ # @!attribute [rw] execution_interval
230
+ # @return [Fixnum] Number of seconds after the task completes before the
231
+ # task is performed again.
232
+ def execution_interval
233
+ synchronize { @execution_interval }
234
+ end
235
+
236
+ # @!attribute [rw] execution_interval
237
+ # @return [Fixnum] Number of seconds after the task completes before the
238
+ # task is performed again.
239
+ def execution_interval=(value)
240
+ raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
241
+
242
+ synchronize { @execution_interval = value }
243
+ end
244
+
245
+ private :post, :<<
246
+
247
+ private
248
+
249
+ def ns_initialize(opts, &task)
250
+ set_deref_options(opts)
251
+
252
+ self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
253
+ if opts[:timeout] || opts[:timeout_interval]
254
+ warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
255
+ end
256
+ @run_now = opts[:now] || opts[:run_now]
257
+ @executor = Concurrent::SafeTaskExecutor.new(task)
258
+ @running = Concurrent::AtomicBoolean.new(false)
259
+ @value = nil
260
+
261
+ self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
262
+ end
263
+
264
+ # @!visibility private
265
+ def ns_shutdown_execution
266
+ @running.make_false
267
+ super
268
+ end
269
+
270
+ # @!visibility private
271
+ def ns_kill_execution
272
+ @running.make_false
273
+ super
274
+ end
275
+
276
+ # @!visibility private
277
+ def schedule_next_task(interval = execution_interval)
278
+ exec_task = ->(completion) { execute_task(completion) }
279
+ Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
280
+ nil
281
+ end
282
+
283
+ # @!visibility private
284
+ def execute_task(completion)
285
+ return nil unless @running.true?
286
+
287
+ _success, value, reason = @executor.execute(self)
288
+ if completion.try?
289
+ self.value = value
290
+ schedule_next_task
291
+ time = Time.now
292
+ observers.notify_observers do
293
+ [time, self.value, reason]
294
+ end
295
+ end
296
+ nil
297
+ end
298
+ end
299
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Handles timing of things
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ module Timing
8
+ module_function
9
+
10
+ #
11
+ # Used for timing method calls
12
+ #
13
+ #
14
+ # @return [yield return, Float]
15
+ #
16
+ def timed
17
+ start_time = time_source.call
18
+
19
+ [yield, time_source.call - start_time]
20
+ end
21
+
22
+ #
23
+ # Used to get a current representation of time as Integer
24
+ #
25
+ #
26
+ # @return [Integer]
27
+ #
28
+ def time_source
29
+ -> { (clock_stamp * 1000).to_i }
30
+ end
31
+
32
+ #
33
+ # Returns the current time as float
34
+ #
35
+ # @see SidekiqUniqueJobs.now_f
36
+ #
37
+ # @return [Float]
38
+ #
39
+ def now_f
40
+ SidekiqUniqueJobs.now_f
41
+ end
42
+
43
+ #
44
+ # Returns a float representation of the current time.
45
+ # Either from Process or Time
46
+ #
47
+ #
48
+ # @return [Float]
49
+ #
50
+ def clock_stamp
51
+ if Process.const_defined?(:CLOCK_MONOTONIC)
52
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
53
+ else
54
+ now_f
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Utility module to help manage unique keys in redis.
5
+ # Useful for deleting keys that for whatever reason wasn't deleted
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ module Unlockable
9
+ module_function
10
+
11
+ # Unlocks a job.
12
+ # @param [Hash] item a Sidekiq job hash
13
+ def unlock(item)
14
+ SidekiqUniqueJobs::Job.add_digest(item)
15
+ SidekiqUniqueJobs::Locksmith.new(item).unlock
16
+ end
17
+
18
+ # Unlocks a job.
19
+ # @param [Hash] item a Sidekiq job hash
20
+ def unlock!(item)
21
+ SidekiqUniqueJobs::Job.add_digest(item)
22
+ SidekiqUniqueJobs::Locksmith.new(item).unlock!
23
+ end
24
+
25
+ # Deletes a lock unless it has ttl
26
+ #
27
+ # This is good for situations when a job is locked by another item
28
+ # @param [Hash] item a Sidekiq job hash
29
+ def delete(item)
30
+ SidekiqUniqueJobs::Job.add_digest(item)
31
+ SidekiqUniqueJobs::Locksmith.new(item).delete
32
+ end
33
+
34
+ # Deletes a lock regardless of if it was locked or has ttl.
35
+ #
36
+ # This is good for situations when a job is locked by another item
37
+ # @param [Hash] item a Sidekiq job hash
38
+ def delete!(item)
39
+ SidekiqUniqueJobs::Job.add_digest(item)
40
+ SidekiqUniqueJobs::Locksmith.new(item).delete!
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class UpdateVersion sets the right version in redis
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class UpdateVersion
10
+ #
11
+ # Sets the right versions in redis
12
+ #
13
+ # @note the version isn't used yet but will be for automatic upgrades
14
+ #
15
+ # @return [true] when version changed
16
+ #
17
+ def self.call
18
+ Script::Caller.call_script(
19
+ :update_version,
20
+ keys: [LIVE_VERSION, DEAD_VERSION],
21
+ argv: [SidekiqUniqueJobs.version],
22
+ )
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,152 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Upgrades locks between gem version upgrades
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class UpgradeLocks
10
+ #
11
+ # @return [Integer] the number of keys to batch upgrade
12
+ BATCH_SIZE = 100
13
+ #
14
+ # @return [Array<String>] suffixes for old version
15
+ OLD_SUFFIXES = %w[
16
+ GRABBED
17
+ AVAILABLE
18
+ EXISTS
19
+ VERSION
20
+ ].freeze
21
+
22
+ include SidekiqUniqueJobs::Logging
23
+ include SidekiqUniqueJobs::Connection
24
+
25
+ #
26
+ # Performs upgrade of old locks
27
+ #
28
+ #
29
+ # @return [Integer] the number of upgrades locks
30
+ #
31
+ def self.call
32
+ redis do |conn|
33
+ new(conn).call
34
+ end
35
+ end
36
+
37
+ attr_reader :conn
38
+
39
+ def initialize(conn)
40
+ @count = 0
41
+ @conn = conn
42
+ redis_version # Avoid pipelined calling redis_version and getting a future.
43
+ end
44
+
45
+ #
46
+ # Performs upgrade of old locks
47
+ #
48
+ #
49
+ # @return [Integer] the number of upgrades locks
50
+ #
51
+ def call
52
+ with_logging_context do
53
+ return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
54
+ # TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
55
+ return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
56
+
57
+ log_info("Start - Upgrading Locks")
58
+
59
+ # upgrade_v6_locks
60
+ # delete_unused_v6_keys
61
+ # delete_supporting_v6_keys
62
+
63
+ conn.hset(upgraded_key, version, now_f)
64
+ log_info("Done - Upgrading Locks")
65
+ end
66
+
67
+ @count
68
+ end
69
+
70
+ private
71
+
72
+ def upgraded_key
73
+ @upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
74
+ end
75
+
76
+ def upgrade_v6_locks
77
+ log_info("Start - Converting v6 locks to v7")
78
+ conn.scan(match: "*:GRABBED", count: BATCH_SIZE).each do |grabbed_key|
79
+ upgrade_v6_lock(grabbed_key)
80
+ @count += 1
81
+ end
82
+
83
+ log_info("Done - Converting v6 locks to v7")
84
+ end
85
+
86
+ def upgrade_v6_lock(grabbed_key)
87
+ locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
88
+ digest = grabbed_key.gsub(":GRABBED", "")
89
+ locks = conn.hgetall(grabbed_key)
90
+
91
+ conn.pipelined do |pipeline|
92
+ pipeline.hmset(locked_key, *locks.to_a)
93
+ pipeline.zadd(DIGESTS, locks.values.first, digest)
94
+ end
95
+ end
96
+
97
+ def delete_unused_v6_keys
98
+ log_info("Start - Deleting v6 keys")
99
+ OLD_SUFFIXES.each do |suffix|
100
+ delete_suffix(suffix)
101
+ end
102
+ log_info("Done - Deleting v6 keys")
103
+ end
104
+
105
+ def delete_supporting_v6_keys
106
+ batch_delete("unique:keys")
107
+ end
108
+
109
+ def delete_suffix(suffix)
110
+ batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
111
+ batch_delete(*keys)
112
+ end
113
+ end
114
+
115
+ def batch_delete(*keys)
116
+ return if keys.empty?
117
+
118
+ conn.pipelined do |pipeline|
119
+ pipeline.unlink(*keys)
120
+ end
121
+ end
122
+
123
+ def batch_scan(match:, count:)
124
+ cursor = "0"
125
+ loop do
126
+ cursor, values = conn.scan(cursor, match: match, count: count)
127
+ yield values
128
+ break if cursor == "0"
129
+ end
130
+ end
131
+
132
+ def version
133
+ SidekiqUniqueJobs.version
134
+ end
135
+
136
+ def now_f
137
+ SidekiqUniqueJobs.now_f
138
+ end
139
+
140
+ def redis_version
141
+ @redis_version ||= SidekiqUniqueJobs.config.redis_version
142
+ end
143
+
144
+ def logging_context
145
+ if logger_context_hash?
146
+ { "uniquejobs" => :upgrade_locks }
147
+ else
148
+ "uniquejobs-upgrade_locks"
149
+ end
150
+ end
151
+ end
152
+ end
@@ -1,3 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module SidekiqUniqueJobs
2
- VERSION = '3.0.11'
4
+ #
5
+ # @return [String] the current SidekiqUniqueJobs version
6
+ VERSION = "8.0.10"
3
7
  end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Handles checking if a version is compliant with given constraint
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class VersionCheck
10
+ PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength, Lint/MixedRegexpCaptureTypes
11
+
12
+ #
13
+ # Checks if a version is constraint is satisfied
14
+ #
15
+ # @example A satisfied constraint
16
+ # VersionCheck.satisfied?("5.0.0", ">= 4.0.0") #=> true
17
+ #
18
+ # @example An unsatisfied constraint
19
+ # VersionCheck.satisfied?("5.0.0", "<= 4.0.0") #=> false
20
+ #
21
+ #
22
+ # @param [String] version a version string `5.0.0`
23
+ # @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
24
+ #
25
+ # @return [true, false] <description>
26
+ #
27
+ def self.satisfied?(version, constraint)
28
+ new(version, constraint).satisfied?
29
+ end
30
+
31
+ #
32
+ # Checks if a version is constraint is unfulfilled
33
+ #
34
+ # @example A satisfied constraint
35
+ # VersionCheck.unfulfilled?("5.0.0", ">= 4.0.0") #=> false
36
+ #
37
+ # @example An unfulfilled constraint
38
+ # VersionCheck.unfulfilled?("5.0.0", "<= 4.0.0") #=> true
39
+ #
40
+ #
41
+ # @param [String] version a version string `5.0.0`
42
+ # @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
43
+ #
44
+ # @return [true, false] <description>
45
+ #
46
+ def self.unfulfilled?(version, constraint)
47
+ !satisfied?(version, constraint)
48
+ end
49
+
50
+ #
51
+ # @!attribute [r] version
52
+ # @return [String] a version string `5.0.0`
53
+ attr_reader :version
54
+ #
55
+ # @!attribute [r] match
56
+ # @return [String] a version constraint `>= 5.0.0 <= 5.1.1`
57
+ attr_reader :match
58
+
59
+ #
60
+ # Initialize a new VersionCheck instance
61
+ #
62
+ # @param [String] version a version string `5.0.0`
63
+ # @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
64
+ #
65
+ def initialize(version, constraint)
66
+ @version = Gem::Version.new(version)
67
+ @match = PATTERN.match(constraint.to_s)
68
+
69
+ raise ArgumentError, "A version (eg. 5.0) is required to compare against" unless @version
70
+ raise ArgumentError, "At least one operator and version is required (eg. >= 5.1)" unless constraint
71
+ end
72
+
73
+ #
74
+ # Checks if all constraints were met
75
+ #
76
+ #
77
+ # @return [true,false]
78
+ #
79
+ def satisfied?
80
+ constraints.all? do |expected, operator|
81
+ compare(expected, operator)
82
+ end
83
+ end
84
+
85
+ private
86
+
87
+ def compare(expected, operator)
88
+ Gem::Version.new(version).send(operator, Gem::Version.new(expected))
89
+ end
90
+
91
+ def constraints
92
+ result = { version_one => operator_one }
93
+ result[version_two] = operator_two if version_two
94
+
95
+ result
96
+ end
97
+
98
+ def version_one
99
+ match[:version1]
100
+ end
101
+
102
+ def operator_one
103
+ match[:operator1]
104
+ end
105
+
106
+ def version_two
107
+ match[:version2]
108
+ end
109
+
110
+ def operator_two
111
+ match[:operator2]
112
+ end
113
+ end
114
+ end