sidekiq-unique-jobs 3.0.11 → 8.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +2163 -25
- data/LICENSE.txt +21 -0
- data/README.md +984 -47
- data/bin/uniquejobs +7 -0
- data/lib/sidekiq-unique-jobs.rb +2 -36
- data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
- data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
- data/lib/sidekiq_unique_jobs/cli.rb +95 -0
- data/lib/sidekiq_unique_jobs/config.rb +306 -33
- data/lib/sidekiq_unique_jobs/connection.rb +20 -0
- data/lib/sidekiq_unique_jobs/constants.rb +55 -0
- data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
- data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
- data/lib/sidekiq_unique_jobs/digests.rb +134 -0
- data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
- data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
- data/lib/sidekiq_unique_jobs/job.rb +63 -0
- data/lib/sidekiq_unique_jobs/json.rb +47 -0
- data/lib/sidekiq_unique_jobs/key.rb +98 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
- data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
- data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
- data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
- data/lib/sidekiq_unique_jobs/lock.rb +348 -0
- data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
- data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
- data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
- data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
- data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
- data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
- data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
- data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
- data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
- data/lib/sidekiq_unique_jobs/logging.rb +236 -0
- data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
- data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
- data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
- data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
- data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
- data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
- data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
- data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
- data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
- data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
- data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
- data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
- data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
- data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
- data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
- data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
- data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
- data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
- data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
- data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
- data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
- data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
- data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
- data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
- data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
- data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
- data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
- data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
- data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
- data/lib/sidekiq_unique_jobs/redis.rb +11 -0
- data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
- data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
- data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
- data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
- data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
- data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
- data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
- data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
- data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
- data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
- data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
- data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
- data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
- data/lib/sidekiq_unique_jobs/script.rb +46 -0
- data/lib/sidekiq_unique_jobs/server.rb +62 -0
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
- data/lib/sidekiq_unique_jobs/testing.rb +132 -9
- data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
- data/lib/sidekiq_unique_jobs/timing.rb +58 -0
- data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
- data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
- data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
- data/lib/sidekiq_unique_jobs/version.rb +5 -1
- data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
- data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
- data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
- data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
- data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
- data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
- data/lib/sidekiq_unique_jobs/web.rb +109 -0
- data/lib/sidekiq_unique_jobs.rb +83 -0
- data/lib/tasks/changelog.rake +23 -0
- metadata +157 -126
- data/.gitignore +0 -10
- data/.rspec +0 -3
- data/.rubocop.yml +0 -36
- data/.travis.yml +0 -25
- data/Appraisals +0 -20
- data/Gemfile +0 -5
- data/LICENSE +0 -22
- data/Rakefile +0 -11
- data/gemfiles/sidekiq_2.15.gemfile +0 -9
- data/gemfiles/sidekiq_2.16.gemfile +0 -9
- data/gemfiles/sidekiq_2.17.gemfile +0 -9
- data/gemfiles/sidekiq_3.0.gemfile +0 -9
- data/gemfiles/sidekiq_develop.gemfile +0 -9
- data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
- data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
- data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
- data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
- data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
- data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
- data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
- data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
- data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
- data/sidekiq-unique-jobs.gemspec +0 -27
- data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
- data/spec/lib/client_spec.rb +0 -173
- data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
- data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
- data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
- data/spec/lib/unlock_order_spec.rb +0 -64
- data/spec/spec_helper.rb +0 -37
- data/spec/support/my_worker.rb +0 -13
- data/spec/support/sidekiq_meta.rb +0 -17
- data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,299 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent/collection/copy_on_notify_observer_set"
|
4
|
+
require "concurrent/concern/dereferenceable"
|
5
|
+
require "concurrent/concern/observable"
|
6
|
+
require "concurrent/atomic/atomic_boolean"
|
7
|
+
require "concurrent/executor/executor_service"
|
8
|
+
require "concurrent/executor/ruby_executor_service"
|
9
|
+
require "concurrent/executor/safe_task_executor"
|
10
|
+
require "concurrent/scheduled_task"
|
11
|
+
|
12
|
+
module SidekiqUniqueJobs
|
13
|
+
# A very common concurrency pattern is to run a thread that performs a task at
|
14
|
+
# regular intervals. The thread that performs the task sleeps for the given
|
15
|
+
# interval then wakes up and performs the task. Lather, rinse, repeat... This
|
16
|
+
# pattern causes two problems. First, it is difficult to test the business
|
17
|
+
# logic of the task because the task itself is tightly coupled with the
|
18
|
+
# concurrency logic. Second, an exception raised while performing the task can
|
19
|
+
# cause the entire thread to abend. In a long-running application where the
|
20
|
+
# task thread is intended to run for days/weeks/years a crashed task thread
|
21
|
+
# can pose a significant problem. `TimerTask` alleviates both problems.
|
22
|
+
#
|
23
|
+
# When a `TimerTask` is launched it starts a thread for monitoring the
|
24
|
+
# execution interval. The `TimerTask` thread does not perform the task,
|
25
|
+
# however. Instead, the TimerTask launches the task on a separate thread.
|
26
|
+
# Should the task experience an unrecoverable crash only the task thread will
|
27
|
+
# crash. This makes the `TimerTask` very fault tolerant. Additionally, the
|
28
|
+
# `TimerTask` thread can respond to the success or failure of the task,
|
29
|
+
# performing logging or ancillary operations.
|
30
|
+
#
|
31
|
+
# One other advantage of `TimerTask` is that it forces the business logic to
|
32
|
+
# be completely decoupled from the concurrency logic. The business logic can
|
33
|
+
# be tested separately then passed to the `TimerTask` for scheduling and
|
34
|
+
# running.
|
35
|
+
#
|
36
|
+
# In some cases it may be necessary for a `TimerTask` to affect its own
|
37
|
+
# execution cycle. To facilitate this, a reference to the TimerTask instance
|
38
|
+
# is passed as an argument to the provided block every time the task is
|
39
|
+
# executed.
|
40
|
+
#
|
41
|
+
# The `TimerTask` class includes the `Dereferenceable` mixin module so the
|
42
|
+
# result of the last execution is always available via the `#value` method.
|
43
|
+
# Dereferencing options can be passed to the `TimerTask` during construction or
|
44
|
+
# at any later time using the `#set_deref_options` method.
|
45
|
+
#
|
46
|
+
# `TimerTask` supports notification through the Ruby standard library
|
47
|
+
# {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
|
48
|
+
# Observable} module. On execution the `TimerTask` will notify the observers
|
49
|
+
# with three arguments: time of execution, the result of the block (or nil on
|
50
|
+
# failure), and any raised exceptions (or nil on success).
|
51
|
+
#
|
52
|
+
# @!macro copy_options
|
53
|
+
#
|
54
|
+
# @example Basic usage
|
55
|
+
# task = Concurrent::TimerTask.new{ puts 'Boom!' }
|
56
|
+
# task.execute
|
57
|
+
#
|
58
|
+
# task.execution_interval #=> 60 (default)
|
59
|
+
#
|
60
|
+
# # wait 60 seconds...
|
61
|
+
# #=> 'Boom!'
|
62
|
+
#
|
63
|
+
# task.shutdown #=> true
|
64
|
+
#
|
65
|
+
# @example Configuring `:execution_interval`
|
66
|
+
# task = Concurrent::TimerTask.new(execution_interval: 5) do
|
67
|
+
# puts 'Boom!'
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# task.execution_interval #=> 5
|
71
|
+
#
|
72
|
+
# @example Immediate execution with `:run_now`
|
73
|
+
# task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
|
74
|
+
# task.execute
|
75
|
+
#
|
76
|
+
# #=> 'Boom!'
|
77
|
+
#
|
78
|
+
# @example Last `#value` and `Dereferenceable` mixin
|
79
|
+
# task = Concurrent::TimerTask.new(
|
80
|
+
# dup_on_deref: true,
|
81
|
+
# execution_interval: 5
|
82
|
+
# ){ Time.now }
|
83
|
+
#
|
84
|
+
# task.execute
|
85
|
+
# Time.now #=> 2013-11-07 18:06:50 -0500
|
86
|
+
# sleep(10)
|
87
|
+
# task.value #=> 2013-11-07 18:06:55 -0500
|
88
|
+
#
|
89
|
+
# @example Controlling execution from within the block
|
90
|
+
# timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
|
91
|
+
# task.execution_interval.times{ print 'Boom! ' }
|
92
|
+
# print "\n"
|
93
|
+
# task.execution_interval += 1
|
94
|
+
# if task.execution_interval > 5
|
95
|
+
# puts 'Stopping...'
|
96
|
+
# task.shutdown
|
97
|
+
# end
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
# timer_task.execute # blocking call - this task will stop itself
|
101
|
+
# #=> Boom!
|
102
|
+
# #=> Boom! Boom!
|
103
|
+
# #=> Boom! Boom! Boom!
|
104
|
+
# #=> Boom! Boom! Boom! Boom!
|
105
|
+
# #=> Boom! Boom! Boom! Boom! Boom!
|
106
|
+
# #=> Stopping...
|
107
|
+
#
|
108
|
+
# @example Observation
|
109
|
+
# class TaskObserver
|
110
|
+
# def update(time, result, ex)
|
111
|
+
# if result
|
112
|
+
# print "(#{time}) Execution successfully returned #{result}\n"
|
113
|
+
# else
|
114
|
+
# print "(#{time}) Execution failed with error #{ex}\n"
|
115
|
+
# end
|
116
|
+
# end
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
|
120
|
+
# task.add_observer(TaskObserver.new)
|
121
|
+
# task.execute
|
122
|
+
# sleep 4
|
123
|
+
#
|
124
|
+
# #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
|
125
|
+
# #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
|
126
|
+
# #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
|
127
|
+
# task.shutdown
|
128
|
+
#
|
129
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
|
130
|
+
# task.add_observer(TaskObserver.new)
|
131
|
+
# task.execute
|
132
|
+
#
|
133
|
+
# #=> (2013-10-13 19:07:25 -0400) Execution timed out
|
134
|
+
# #=> (2013-10-13 19:07:27 -0400) Execution timed out
|
135
|
+
# #=> (2013-10-13 19:07:29 -0400) Execution timed out
|
136
|
+
# task.shutdown
|
137
|
+
#
|
138
|
+
# task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
|
139
|
+
# task.add_observer(TaskObserver.new)
|
140
|
+
# task.execute
|
141
|
+
#
|
142
|
+
# #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
|
143
|
+
# #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
|
144
|
+
# #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
|
145
|
+
# task.shutdown
|
146
|
+
#
|
147
|
+
# @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
|
148
|
+
# @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
|
149
|
+
class TimerTask < Concurrent::RubyExecutorService
|
150
|
+
include Concurrent::Concern::Dereferenceable
|
151
|
+
include Concurrent::Concern::Observable
|
152
|
+
|
153
|
+
# Default `:execution_interval` in seconds.
|
154
|
+
EXECUTION_INTERVAL = 60
|
155
|
+
|
156
|
+
# Default `:timeout_interval` in seconds.
|
157
|
+
TIMEOUT_INTERVAL = 30
|
158
|
+
|
159
|
+
# Create a new TimerTask with the given task and configuration.
|
160
|
+
#
|
161
|
+
# @!macro timer_task_initialize
|
162
|
+
# @param [Hash] opts the options defining task execution.
|
163
|
+
# @option opts [Integer] :execution_interval number of seconds between
|
164
|
+
# task executions (default: EXECUTION_INTERVAL)
|
165
|
+
# @option opts [Boolean] :run_now Whether to run the task immediately
|
166
|
+
# upon instantiation or to wait until the first # execution_interval
|
167
|
+
# has passed (default: false)
|
168
|
+
#
|
169
|
+
# @!macro deref_options
|
170
|
+
#
|
171
|
+
# @raise ArgumentError when no block is given.
|
172
|
+
#
|
173
|
+
# @yield to the block after :execution_interval seconds have passed since
|
174
|
+
# the last yield
|
175
|
+
# @yieldparam task a reference to the `TimerTask` instance so that the
|
176
|
+
# block can control its own lifecycle. Necessary since `self` will
|
177
|
+
# refer to the execution context of the block rather than the running
|
178
|
+
# `TimerTask`.
|
179
|
+
#
|
180
|
+
# @return [TimerTask] the new `TimerTask`
|
181
|
+
def initialize(opts = {}, &task)
|
182
|
+
raise ArgumentError, "no block given" unless task
|
183
|
+
|
184
|
+
super
|
185
|
+
set_deref_options opts
|
186
|
+
end
|
187
|
+
|
188
|
+
# Is the executor running?
|
189
|
+
#
|
190
|
+
# @return [Boolean] `true` when running, `false` when shutting down or shutdown
|
191
|
+
def running?
|
192
|
+
@running.true?
|
193
|
+
end
|
194
|
+
|
195
|
+
# Execute a previously created `TimerTask`.
|
196
|
+
#
|
197
|
+
# @return [TimerTask] a reference to `self`
|
198
|
+
#
|
199
|
+
# @example Instance and execute in separate steps
|
200
|
+
# task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
|
201
|
+
# task.running? #=> false
|
202
|
+
# task.execute
|
203
|
+
# task.running? #=> true
|
204
|
+
#
|
205
|
+
# @example Instance and execute in one line
|
206
|
+
# task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
|
207
|
+
# task.running? #=> true
|
208
|
+
def execute
|
209
|
+
synchronize do
|
210
|
+
if @running.false?
|
211
|
+
@running.make_true
|
212
|
+
schedule_next_task(@run_now ? 0 : @execution_interval)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
self
|
216
|
+
end
|
217
|
+
|
218
|
+
# Create and execute a new `TimerTask`.
|
219
|
+
#
|
220
|
+
# @!macro timer_task_initialize
|
221
|
+
#
|
222
|
+
# @example
|
223
|
+
# task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
|
224
|
+
# task.running? #=> true
|
225
|
+
def self.execute(opts = {}, &task)
|
226
|
+
TimerTask.new(opts, &task).execute
|
227
|
+
end
|
228
|
+
|
229
|
+
# @!attribute [rw] execution_interval
|
230
|
+
# @return [Fixnum] Number of seconds after the task completes before the
|
231
|
+
# task is performed again.
|
232
|
+
def execution_interval
|
233
|
+
synchronize { @execution_interval }
|
234
|
+
end
|
235
|
+
|
236
|
+
# @!attribute [rw] execution_interval
|
237
|
+
# @return [Fixnum] Number of seconds after the task completes before the
|
238
|
+
# task is performed again.
|
239
|
+
def execution_interval=(value)
|
240
|
+
raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
|
241
|
+
|
242
|
+
synchronize { @execution_interval = value }
|
243
|
+
end
|
244
|
+
|
245
|
+
private :post, :<<
|
246
|
+
|
247
|
+
private
|
248
|
+
|
249
|
+
def ns_initialize(opts, &task)
|
250
|
+
set_deref_options(opts)
|
251
|
+
|
252
|
+
self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
|
253
|
+
if opts[:timeout] || opts[:timeout_interval]
|
254
|
+
warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
|
255
|
+
end
|
256
|
+
@run_now = opts[:now] || opts[:run_now]
|
257
|
+
@executor = Concurrent::SafeTaskExecutor.new(task)
|
258
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
259
|
+
@value = nil
|
260
|
+
|
261
|
+
self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
|
262
|
+
end
|
263
|
+
|
264
|
+
# @!visibility private
|
265
|
+
def ns_shutdown_execution
|
266
|
+
@running.make_false
|
267
|
+
super
|
268
|
+
end
|
269
|
+
|
270
|
+
# @!visibility private
|
271
|
+
def ns_kill_execution
|
272
|
+
@running.make_false
|
273
|
+
super
|
274
|
+
end
|
275
|
+
|
276
|
+
# @!visibility private
|
277
|
+
def schedule_next_task(interval = execution_interval)
|
278
|
+
exec_task = ->(completion) { execute_task(completion) }
|
279
|
+
Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
|
280
|
+
nil
|
281
|
+
end
|
282
|
+
|
283
|
+
# @!visibility private
|
284
|
+
def execute_task(completion)
|
285
|
+
return nil unless @running.true?
|
286
|
+
|
287
|
+
_success, value, reason = @executor.execute(self)
|
288
|
+
if completion.try?
|
289
|
+
self.value = value
|
290
|
+
schedule_next_task
|
291
|
+
time = Time.now
|
292
|
+
observers.notify_observers do
|
293
|
+
[time, self.value, reason]
|
294
|
+
end
|
295
|
+
end
|
296
|
+
nil
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
# Handles timing of things
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
|
+
module Timing
|
8
|
+
module_function
|
9
|
+
|
10
|
+
#
|
11
|
+
# Used for timing method calls
|
12
|
+
#
|
13
|
+
#
|
14
|
+
# @return [yield return, Float]
|
15
|
+
#
|
16
|
+
def timed
|
17
|
+
start_time = time_source.call
|
18
|
+
|
19
|
+
[yield, time_source.call - start_time]
|
20
|
+
end
|
21
|
+
|
22
|
+
#
|
23
|
+
# Used to get a current representation of time as Integer
|
24
|
+
#
|
25
|
+
#
|
26
|
+
# @return [Integer]
|
27
|
+
#
|
28
|
+
def time_source
|
29
|
+
-> { (clock_stamp * 1000).to_i }
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# Returns the current time as float
|
34
|
+
#
|
35
|
+
# @see SidekiqUniqueJobs.now_f
|
36
|
+
#
|
37
|
+
# @return [Float]
|
38
|
+
#
|
39
|
+
def now_f
|
40
|
+
SidekiqUniqueJobs.now_f
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Returns a float representation of the current time.
|
45
|
+
# Either from Process or Time
|
46
|
+
#
|
47
|
+
#
|
48
|
+
# @return [Float]
|
49
|
+
#
|
50
|
+
def clock_stamp
|
51
|
+
if Process.const_defined?(:CLOCK_MONOTONIC)
|
52
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
53
|
+
else
|
54
|
+
now_f
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
# Utility module to help manage unique keys in redis.
|
5
|
+
# Useful for deleting keys that for whatever reason wasn't deleted
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
module Unlockable
|
9
|
+
module_function
|
10
|
+
|
11
|
+
# Unlocks a job.
|
12
|
+
# @param [Hash] item a Sidekiq job hash
|
13
|
+
def unlock(item)
|
14
|
+
SidekiqUniqueJobs::Job.add_digest(item)
|
15
|
+
SidekiqUniqueJobs::Locksmith.new(item).unlock
|
16
|
+
end
|
17
|
+
|
18
|
+
# Unlocks a job.
|
19
|
+
# @param [Hash] item a Sidekiq job hash
|
20
|
+
def unlock!(item)
|
21
|
+
SidekiqUniqueJobs::Job.add_digest(item)
|
22
|
+
SidekiqUniqueJobs::Locksmith.new(item).unlock!
|
23
|
+
end
|
24
|
+
|
25
|
+
# Deletes a lock unless it has ttl
|
26
|
+
#
|
27
|
+
# This is good for situations when a job is locked by another item
|
28
|
+
# @param [Hash] item a Sidekiq job hash
|
29
|
+
def delete(item)
|
30
|
+
SidekiqUniqueJobs::Job.add_digest(item)
|
31
|
+
SidekiqUniqueJobs::Locksmith.new(item).delete
|
32
|
+
end
|
33
|
+
|
34
|
+
# Deletes a lock regardless of if it was locked or has ttl.
|
35
|
+
#
|
36
|
+
# This is good for situations when a job is locked by another item
|
37
|
+
# @param [Hash] item a Sidekiq job hash
|
38
|
+
def delete!(item)
|
39
|
+
SidekiqUniqueJobs::Job.add_digest(item)
|
40
|
+
SidekiqUniqueJobs::Locksmith.new(item).delete!
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
#
|
5
|
+
# Class UpdateVersion sets the right version in redis
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
#
|
9
|
+
class UpdateVersion
|
10
|
+
#
|
11
|
+
# Sets the right versions in redis
|
12
|
+
#
|
13
|
+
# @note the version isn't used yet but will be for automatic upgrades
|
14
|
+
#
|
15
|
+
# @return [true] when version changed
|
16
|
+
#
|
17
|
+
def self.call
|
18
|
+
Script::Caller.call_script(
|
19
|
+
:update_version,
|
20
|
+
keys: [LIVE_VERSION, DEAD_VERSION],
|
21
|
+
argv: [SidekiqUniqueJobs.version],
|
22
|
+
)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,152 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
#
|
5
|
+
# Upgrades locks between gem version upgrades
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
#
|
9
|
+
class UpgradeLocks
|
10
|
+
#
|
11
|
+
# @return [Integer] the number of keys to batch upgrade
|
12
|
+
BATCH_SIZE = 100
|
13
|
+
#
|
14
|
+
# @return [Array<String>] suffixes for old version
|
15
|
+
OLD_SUFFIXES = %w[
|
16
|
+
GRABBED
|
17
|
+
AVAILABLE
|
18
|
+
EXISTS
|
19
|
+
VERSION
|
20
|
+
].freeze
|
21
|
+
|
22
|
+
include SidekiqUniqueJobs::Logging
|
23
|
+
include SidekiqUniqueJobs::Connection
|
24
|
+
|
25
|
+
#
|
26
|
+
# Performs upgrade of old locks
|
27
|
+
#
|
28
|
+
#
|
29
|
+
# @return [Integer] the number of upgrades locks
|
30
|
+
#
|
31
|
+
def self.call
|
32
|
+
redis do |conn|
|
33
|
+
new(conn).call
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
attr_reader :conn
|
38
|
+
|
39
|
+
def initialize(conn)
|
40
|
+
@count = 0
|
41
|
+
@conn = conn
|
42
|
+
redis_version # Avoid pipelined calling redis_version and getting a future.
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# Performs upgrade of old locks
|
47
|
+
#
|
48
|
+
#
|
49
|
+
# @return [Integer] the number of upgrades locks
|
50
|
+
#
|
51
|
+
def call
|
52
|
+
with_logging_context do
|
53
|
+
return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
|
54
|
+
# TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
|
55
|
+
return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
|
56
|
+
|
57
|
+
log_info("Start - Upgrading Locks")
|
58
|
+
|
59
|
+
# upgrade_v6_locks
|
60
|
+
# delete_unused_v6_keys
|
61
|
+
# delete_supporting_v6_keys
|
62
|
+
|
63
|
+
conn.hset(upgraded_key, version, now_f)
|
64
|
+
log_info("Done - Upgrading Locks")
|
65
|
+
end
|
66
|
+
|
67
|
+
@count
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def upgraded_key
|
73
|
+
@upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
|
74
|
+
end
|
75
|
+
|
76
|
+
def upgrade_v6_locks
|
77
|
+
log_info("Start - Converting v6 locks to v7")
|
78
|
+
conn.scan(match: "*:GRABBED", count: BATCH_SIZE).each do |grabbed_key|
|
79
|
+
upgrade_v6_lock(grabbed_key)
|
80
|
+
@count += 1
|
81
|
+
end
|
82
|
+
|
83
|
+
log_info("Done - Converting v6 locks to v7")
|
84
|
+
end
|
85
|
+
|
86
|
+
def upgrade_v6_lock(grabbed_key)
|
87
|
+
locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
|
88
|
+
digest = grabbed_key.gsub(":GRABBED", "")
|
89
|
+
locks = conn.hgetall(grabbed_key)
|
90
|
+
|
91
|
+
conn.pipelined do |pipeline|
|
92
|
+
pipeline.hmset(locked_key, *locks.to_a)
|
93
|
+
pipeline.zadd(DIGESTS, locks.values.first, digest)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def delete_unused_v6_keys
|
98
|
+
log_info("Start - Deleting v6 keys")
|
99
|
+
OLD_SUFFIXES.each do |suffix|
|
100
|
+
delete_suffix(suffix)
|
101
|
+
end
|
102
|
+
log_info("Done - Deleting v6 keys")
|
103
|
+
end
|
104
|
+
|
105
|
+
def delete_supporting_v6_keys
|
106
|
+
batch_delete("unique:keys")
|
107
|
+
end
|
108
|
+
|
109
|
+
def delete_suffix(suffix)
|
110
|
+
batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
|
111
|
+
batch_delete(*keys)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def batch_delete(*keys)
|
116
|
+
return if keys.empty?
|
117
|
+
|
118
|
+
conn.pipelined do |pipeline|
|
119
|
+
pipeline.unlink(*keys)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def batch_scan(match:, count:)
|
124
|
+
cursor = "0"
|
125
|
+
loop do
|
126
|
+
cursor, values = conn.scan(cursor, match: match, count: count)
|
127
|
+
yield values
|
128
|
+
break if cursor == "0"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def version
|
133
|
+
SidekiqUniqueJobs.version
|
134
|
+
end
|
135
|
+
|
136
|
+
def now_f
|
137
|
+
SidekiqUniqueJobs.now_f
|
138
|
+
end
|
139
|
+
|
140
|
+
def redis_version
|
141
|
+
@redis_version ||= SidekiqUniqueJobs.config.redis_version
|
142
|
+
end
|
143
|
+
|
144
|
+
def logging_context
|
145
|
+
if logger_context_hash?
|
146
|
+
{ "uniquejobs" => :upgrade_locks }
|
147
|
+
else
|
148
|
+
"uniquejobs-upgrade_locks"
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
#
|
5
|
+
# Handles checking if a version is compliant with given constraint
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
#
|
9
|
+
class VersionCheck
|
10
|
+
PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength, Lint/MixedRegexpCaptureTypes
|
11
|
+
|
12
|
+
#
|
13
|
+
# Checks if a version is constraint is satisfied
|
14
|
+
#
|
15
|
+
# @example A satisfied constraint
|
16
|
+
# VersionCheck.satisfied?("5.0.0", ">= 4.0.0") #=> true
|
17
|
+
#
|
18
|
+
# @example An unsatisfied constraint
|
19
|
+
# VersionCheck.satisfied?("5.0.0", "<= 4.0.0") #=> false
|
20
|
+
#
|
21
|
+
#
|
22
|
+
# @param [String] version a version string `5.0.0`
|
23
|
+
# @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
|
24
|
+
#
|
25
|
+
# @return [true, false] <description>
|
26
|
+
#
|
27
|
+
def self.satisfied?(version, constraint)
|
28
|
+
new(version, constraint).satisfied?
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Checks if a version is constraint is unfulfilled
|
33
|
+
#
|
34
|
+
# @example A satisfied constraint
|
35
|
+
# VersionCheck.unfulfilled?("5.0.0", ">= 4.0.0") #=> false
|
36
|
+
#
|
37
|
+
# @example An unfulfilled constraint
|
38
|
+
# VersionCheck.unfulfilled?("5.0.0", "<= 4.0.0") #=> true
|
39
|
+
#
|
40
|
+
#
|
41
|
+
# @param [String] version a version string `5.0.0`
|
42
|
+
# @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
|
43
|
+
#
|
44
|
+
# @return [true, false] <description>
|
45
|
+
#
|
46
|
+
def self.unfulfilled?(version, constraint)
|
47
|
+
!satisfied?(version, constraint)
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# @!attribute [r] version
|
52
|
+
# @return [String] a version string `5.0.0`
|
53
|
+
attr_reader :version
|
54
|
+
#
|
55
|
+
# @!attribute [r] match
|
56
|
+
# @return [String] a version constraint `>= 5.0.0 <= 5.1.1`
|
57
|
+
attr_reader :match
|
58
|
+
|
59
|
+
#
|
60
|
+
# Initialize a new VersionCheck instance
|
61
|
+
#
|
62
|
+
# @param [String] version a version string `5.0.0`
|
63
|
+
# @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
|
64
|
+
#
|
65
|
+
def initialize(version, constraint)
|
66
|
+
@version = Gem::Version.new(version)
|
67
|
+
@match = PATTERN.match(constraint.to_s)
|
68
|
+
|
69
|
+
raise ArgumentError, "A version (eg. 5.0) is required to compare against" unless @version
|
70
|
+
raise ArgumentError, "At least one operator and version is required (eg. >= 5.1)" unless constraint
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Checks if all constraints were met
|
75
|
+
#
|
76
|
+
#
|
77
|
+
# @return [true,false]
|
78
|
+
#
|
79
|
+
def satisfied?
|
80
|
+
constraints.all? do |expected, operator|
|
81
|
+
compare(expected, operator)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def compare(expected, operator)
|
88
|
+
Gem::Version.new(version).send(operator, Gem::Version.new(expected))
|
89
|
+
end
|
90
|
+
|
91
|
+
def constraints
|
92
|
+
result = { version_one => operator_one }
|
93
|
+
result[version_two] = operator_two if version_two
|
94
|
+
|
95
|
+
result
|
96
|
+
end
|
97
|
+
|
98
|
+
def version_one
|
99
|
+
match[:version1]
|
100
|
+
end
|
101
|
+
|
102
|
+
def operator_one
|
103
|
+
match[:operator1]
|
104
|
+
end
|
105
|
+
|
106
|
+
def version_two
|
107
|
+
match[:version2]
|
108
|
+
end
|
109
|
+
|
110
|
+
def operator_two
|
111
|
+
match[:operator2]
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|