sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.codeclimate.yml +17 -9
- data/.gitignore +1 -3
- data/.reek +105 -0
- data/.rubocop.yml +36 -1
- data/.simplecov +7 -2
- data/.travis.yml +11 -10
- data/Appraisals +3 -7
- data/CHANGELOG.md +17 -0
- data/Gemfile +16 -13
- data/Guardfile +55 -0
- data/README.md +85 -73
- data/examples/another_unique_job.rb +13 -0
- data/examples/custom_queue_job.rb +12 -0
- data/examples/custom_queue_job_with_filter_method.rb +13 -0
- data/examples/custom_queue_job_with_filter_proc.rb +16 -0
- data/examples/expiring_job.rb +12 -0
- data/examples/inline_worker.rb +12 -0
- data/examples/just_a_worker.rb +13 -0
- data/examples/long_running_job.rb +12 -0
- data/examples/main_job.rb +13 -0
- data/examples/my_job.rb +12 -0
- data/examples/my_unique_job.rb +16 -0
- data/examples/my_unique_job_with_filter_method.rb +21 -0
- data/examples/my_unique_job_with_filter_proc.rb +19 -0
- data/examples/notify_worker.rb +14 -0
- data/examples/plain_class.rb +13 -0
- data/examples/simple_worker.rb +15 -0
- data/examples/spawn_simple_worker.rb +12 -0
- data/examples/test_class.rb +9 -0
- data/examples/unique_across_workers_job.rb +20 -0
- data/examples/unique_job_with_conditional_parameter.rb +18 -0
- data/examples/unique_job_with_filter_method.rb +18 -0
- data/examples/unique_job_with_nil_unique_args.rb +20 -0
- data/examples/unique_job_with_no_unique_args_method.rb +16 -0
- data/examples/unique_job_withthout_unique_args_parameter.rb +18 -0
- data/examples/unique_on_all_queues_job.rb +16 -0
- data/examples/until_and_while_executing_job.rb +13 -0
- data/examples/until_executed_2_job.rb +24 -0
- data/examples/until_executed_job.rb +25 -0
- data/examples/until_executing_job.rb +11 -0
- data/examples/until_expired_job.rb +12 -0
- data/examples/until_global_expired_job.rb +12 -0
- data/examples/while_executing_job.rb +15 -0
- data/examples/while_executing_reject_job.rb +14 -0
- data/examples/without_argument_job.rb +13 -0
- data/lib/sidekiq-unique-jobs.rb +1 -91
- data/lib/sidekiq/simulator.rb +15 -16
- data/lib/sidekiq_unique_jobs.rb +79 -0
- data/lib/sidekiq_unique_jobs/cli.rb +9 -18
- data/lib/sidekiq_unique_jobs/client/middleware.rb +16 -18
- data/lib/sidekiq_unique_jobs/connection.rb +18 -0
- data/lib/sidekiq_unique_jobs/constants.rb +13 -17
- data/lib/sidekiq_unique_jobs/core_ext.rb +28 -55
- data/lib/sidekiq_unique_jobs/exceptions.rb +30 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +84 -0
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +11 -6
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +6 -58
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +6 -5
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +17 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +20 -34
- data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +78 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing_requeue.rb +20 -0
- data/lib/sidekiq_unique_jobs/locksmith.rb +149 -0
- data/lib/sidekiq_unique_jobs/logging.rb +30 -0
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +27 -41
- data/lib/sidekiq_unique_jobs/scripts.rb +25 -24
- data/lib/sidekiq_unique_jobs/server/middleware.rb +12 -20
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +5 -5
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +42 -0
- data/lib/sidekiq_unique_jobs/testing.rb +40 -50
- data/lib/sidekiq_unique_jobs/timeout.rb +15 -0
- data/lib/sidekiq_unique_jobs/timeout/calculator.rb +49 -0
- data/lib/sidekiq_unique_jobs/unique_args.rb +33 -77
- data/lib/sidekiq_unique_jobs/unlockable.rb +5 -14
- data/lib/sidekiq_unique_jobs/util.rb +28 -90
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/redis/acquire_lock.lua +5 -3
- data/redis/create.lua +58 -0
- data/redis/delete.lua +11 -0
- data/redis/release_stale_locks.lua +90 -0
- data/redis/signal.lua +21 -0
- data/sidekiq-unique-jobs.gemspec +15 -8
- metadata +108 -32
- data/lib/sidekiq_unique_jobs/config.rb +0 -17
- data/lib/sidekiq_unique_jobs/lock.rb +0 -12
- data/lib/sidekiq_unique_jobs/lock/until_timeout.rb +0 -17
- data/lib/sidekiq_unique_jobs/run_lock_failed.rb +0 -3
- data/lib/sidekiq_unique_jobs/script_mock.rb +0 -66
- data/lib/sidekiq_unique_jobs/scripts/acquire_lock.rb +0 -47
- data/lib/sidekiq_unique_jobs/scripts/release_lock.rb +0 -49
- data/lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb +0 -50
- data/lib/sidekiq_unique_jobs/timeout_calculator.rb +0 -67
- data/redis/synchronize.lua +0 -16
@@ -1,17 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SidekiqUniqueJobs
|
4
|
-
class Config < OpenStruct
|
5
|
-
def inline_testing_enabled?
|
6
|
-
testing_enabled? && Sidekiq::Testing.inline?
|
7
|
-
end
|
8
|
-
|
9
|
-
def mocking?
|
10
|
-
redis_test_mode.to_sym == :mock
|
11
|
-
end
|
12
|
-
|
13
|
-
def testing_enabled?
|
14
|
-
Sidekiq.const_defined?(TESTING_CONSTANT, false) && Sidekiq::Testing.enabled?
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'sidekiq_unique_jobs/lock/until_executed'
|
4
|
-
require 'sidekiq_unique_jobs/lock/until_executing'
|
5
|
-
require 'sidekiq_unique_jobs/lock/while_executing'
|
6
|
-
require 'sidekiq_unique_jobs/lock/until_timeout'
|
7
|
-
require 'sidekiq_unique_jobs/lock/until_and_while_executing'
|
8
|
-
|
9
|
-
module SidekiqUniqueJobs
|
10
|
-
module Lock
|
11
|
-
end
|
12
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SidekiqUniqueJobs
|
4
|
-
module Lock
|
5
|
-
class UntilTimeout < UntilExecuted
|
6
|
-
def unlock(scope)
|
7
|
-
return true if scope.to_sym == :server
|
8
|
-
|
9
|
-
raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
|
10
|
-
end
|
11
|
-
|
12
|
-
def execute(_callback)
|
13
|
-
yield if block_given?
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
@@ -1,66 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'pathname'
|
4
|
-
require 'digest/sha1'
|
5
|
-
|
6
|
-
module SidekiqUniqueJobs
|
7
|
-
module ScriptMock
|
8
|
-
module_function
|
9
|
-
|
10
|
-
extend SingleForwardable
|
11
|
-
def_delegator :SidekiqUniqueJobs, :connection
|
12
|
-
|
13
|
-
def call(file_name, redis_pool, options = {})
|
14
|
-
send(file_name, redis_pool, options)
|
15
|
-
end
|
16
|
-
|
17
|
-
def acquire_lock(redis_pool, options = {})
|
18
|
-
connection(redis_pool) do |conn|
|
19
|
-
unique_key = options[:keys][0]
|
20
|
-
job_id = options[:argv][0]
|
21
|
-
expires = options[:argv][1].to_i
|
22
|
-
stored_jid = conn.get(unique_key)
|
23
|
-
|
24
|
-
return (stored_jid == job_id) ? 1 : 0 if stored_jid
|
25
|
-
return 0 unless conn.set(unique_key, job_id, nx: true, ex: expires)
|
26
|
-
|
27
|
-
conn.hsetnx(SidekiqUniqueJobs::HASH_KEY, job_id, unique_key)
|
28
|
-
|
29
|
-
return 1
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def release_lock(redis_pool, options = {})
|
34
|
-
connection(redis_pool) do |conn|
|
35
|
-
unique_key = options[:keys][0]
|
36
|
-
job_id = options[:argv][0]
|
37
|
-
stored_jid = conn.get(unique_key)
|
38
|
-
|
39
|
-
return -1 unless stored_jid
|
40
|
-
return 0 unless stored_jid == job_id || stored_jid == '2'
|
41
|
-
|
42
|
-
conn.del(unique_key)
|
43
|
-
conn.hdel(SidekiqUniqueJobs::HASH_KEY, job_id)
|
44
|
-
|
45
|
-
return 1
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
def synchronize(redis_pool, options = {})
|
50
|
-
connection(redis_pool) do |conn|
|
51
|
-
unique_key = options[:keys][0]
|
52
|
-
time = options[:argv][0].to_i
|
53
|
-
expires = options[:argv][1].to_f
|
54
|
-
|
55
|
-
return 1 if conn.set(unique_key, time + expires, nx: true, ex: expires)
|
56
|
-
|
57
|
-
stored_time = conn.get(unique_key)
|
58
|
-
if stored_time && stored_time < time
|
59
|
-
return 1 if conn.set(unique_key, time + expires, xx: true, ex: expires)
|
60
|
-
end
|
61
|
-
|
62
|
-
return 0
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
@@ -1,47 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SidekiqUniqueJobs
|
4
|
-
module Scripts
|
5
|
-
class AcquireLock
|
6
|
-
extend Forwardable
|
7
|
-
def_delegator SidekiqUniqueJobs, :logger
|
8
|
-
|
9
|
-
def self.execute(redis_pool, unique_key, jid, max_lock_time)
|
10
|
-
new(redis_pool, unique_key, jid, max_lock_time).execute
|
11
|
-
end
|
12
|
-
|
13
|
-
attr_reader :redis_pool, :unique_key, :jid, :max_lock_time
|
14
|
-
|
15
|
-
def initialize(_redis_pool, unique_key, jid, max_lock_time)
|
16
|
-
raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
|
17
|
-
raise JidMissing, 'jid is required' if jid.nil?
|
18
|
-
raise MaxLockTimeMissing, 'max_lock_time is required' if max_lock_time.nil?
|
19
|
-
|
20
|
-
@unique_key = unique_key
|
21
|
-
@jid = jid
|
22
|
-
@max_lock_time = max_lock_time
|
23
|
-
end
|
24
|
-
|
25
|
-
def execute
|
26
|
-
result = Scripts.call(:acquire_lock, redis_pool,
|
27
|
-
keys: [unique_key],
|
28
|
-
argv: [jid, max_lock_time])
|
29
|
-
|
30
|
-
handle_result(result)
|
31
|
-
end
|
32
|
-
|
33
|
-
def handle_result(result)
|
34
|
-
case result
|
35
|
-
when 1
|
36
|
-
logger.debug { "successfully acquired lock #{unique_key} for #{max_lock_time} seconds" }
|
37
|
-
true
|
38
|
-
when 0
|
39
|
-
logger.debug { "failed to acquire lock for #{unique_key}" }
|
40
|
-
false
|
41
|
-
else
|
42
|
-
raise UnexpectedValue, "failed to acquire lock : unexpected return value (#{result})"
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SidekiqUniqueJobs
|
4
|
-
module Scripts
|
5
|
-
class ReleaseLock
|
6
|
-
extend Forwardable
|
7
|
-
def_delegator SidekiqUniqueJobs, :logger
|
8
|
-
|
9
|
-
def self.execute(redis_pool, unique_key, jid)
|
10
|
-
new(redis_pool, unique_key, jid).execute
|
11
|
-
end
|
12
|
-
|
13
|
-
attr_reader :redis_pool, :unique_key, :jid
|
14
|
-
|
15
|
-
def initialize(redis_pool, unique_key, jid)
|
16
|
-
raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
|
17
|
-
raise JidMissing, 'jid is required' if jid.nil?
|
18
|
-
|
19
|
-
@redis_pool = redis_pool
|
20
|
-
@unique_key = unique_key
|
21
|
-
@jid = jid
|
22
|
-
end
|
23
|
-
|
24
|
-
def execute
|
25
|
-
result = Scripts.call(:release_lock, redis_pool,
|
26
|
-
keys: [unique_key],
|
27
|
-
argv: [jid])
|
28
|
-
|
29
|
-
handle_result(result)
|
30
|
-
end
|
31
|
-
|
32
|
-
def handle_result(result)
|
33
|
-
case result
|
34
|
-
when 1
|
35
|
-
logger.debug { "successfully unlocked #{unique_key}" }
|
36
|
-
true
|
37
|
-
when 0
|
38
|
-
logger.debug { "expiring lock #{unique_key} is not owned by #{jid}" }
|
39
|
-
false
|
40
|
-
when -1
|
41
|
-
logger.debug { "#{unique_key} is not a known key" }
|
42
|
-
false
|
43
|
-
else
|
44
|
-
raise UnexpectedValue, "failed to release lock : unexpected return value (#{result})"
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,50 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'sidekiq/testing'
|
4
|
-
|
5
|
-
module Sidekiq
|
6
|
-
module Worker
|
7
|
-
module ClassMethods
|
8
|
-
# Clear all jobs for this worker
|
9
|
-
def clear
|
10
|
-
jobs.each do |job|
|
11
|
-
unlock(job) if Sidekiq::Testing.fake?
|
12
|
-
end
|
13
|
-
|
14
|
-
Sidekiq::Queues[queue].clear
|
15
|
-
jobs.clear
|
16
|
-
end
|
17
|
-
|
18
|
-
unless respond_to?(:execute_job)
|
19
|
-
def execute_job(worker, args)
|
20
|
-
worker.perform(*args)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def unlock(job)
|
25
|
-
SidekiqUniqueJobs::Unlockable.unlock(job)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
module Overrides
|
30
|
-
def self.included(base)
|
31
|
-
base.extend Testing
|
32
|
-
base.class_eval do
|
33
|
-
class << self
|
34
|
-
alias_method :clear_all_orig, :clear_all
|
35
|
-
alias_method :clear_all, :clear_all_ext
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
module Testing
|
41
|
-
def clear_all_ext
|
42
|
-
SidekiqUniqueJobs::Util.del('*', 1000, false) unless SidekiqUniqueJobs.mocked?
|
43
|
-
clear_all_orig
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
include Overrides
|
49
|
-
end
|
50
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SidekiqUniqueJobs
|
4
|
-
class TimeoutCalculator
|
5
|
-
def self.for_item(item)
|
6
|
-
new(item)
|
7
|
-
end
|
8
|
-
|
9
|
-
def initialize(item)
|
10
|
-
@item = item
|
11
|
-
end
|
12
|
-
|
13
|
-
def time_until_scheduled
|
14
|
-
scheduled = item[AT_KEY]
|
15
|
-
return 0 unless scheduled
|
16
|
-
(Time.at(scheduled) - Time.now.utc).to_i
|
17
|
-
end
|
18
|
-
|
19
|
-
def seconds
|
20
|
-
raise NotImplementedError
|
21
|
-
end
|
22
|
-
|
23
|
-
def worker_class_queue_lock_expiration
|
24
|
-
worker_class_expiration_for QUEUE_LOCK_TIMEOUT_KEY
|
25
|
-
end
|
26
|
-
|
27
|
-
def worker_class_run_lock_expiration
|
28
|
-
worker_class_expiration_for RUN_LOCK_TIMEOUT_KEY
|
29
|
-
end
|
30
|
-
|
31
|
-
def worker_class
|
32
|
-
@worker_class ||= SidekiqUniqueJobs.worker_class_constantize(item[CLASS_KEY])
|
33
|
-
end
|
34
|
-
|
35
|
-
private
|
36
|
-
|
37
|
-
def worker_class_expiration_for(key)
|
38
|
-
return unless worker_class.respond_to?(:get_sidekiq_options)
|
39
|
-
worker_class.get_sidekiq_options[key]
|
40
|
-
end
|
41
|
-
|
42
|
-
attr_reader :item
|
43
|
-
end
|
44
|
-
|
45
|
-
class RunLockTimeoutCalculator < TimeoutCalculator
|
46
|
-
def seconds
|
47
|
-
@seconds ||= (
|
48
|
-
worker_class_run_lock_expiration ||
|
49
|
-
SidekiqUniqueJobs.config.default_run_lock_expiration
|
50
|
-
).to_i
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
class QueueLockTimeoutCalculator < TimeoutCalculator
|
55
|
-
def seconds
|
56
|
-
queue_lock_expiration + time_until_scheduled
|
57
|
-
end
|
58
|
-
|
59
|
-
def queue_lock_expiration
|
60
|
-
@queue_lock_expiration ||=
|
61
|
-
(
|
62
|
-
worker_class_queue_lock_expiration ||
|
63
|
-
SidekiqUniqueJobs.config.default_queue_lock_expiration
|
64
|
-
).to_i
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
data/redis/synchronize.lua
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
local unique_key = KEYS[1]
|
2
|
-
local time = ARGV[1]
|
3
|
-
local expires = ARGV[2]
|
4
|
-
|
5
|
-
if redis.pcall('set', unique_key, time + expires, 'nx', 'ex', expires) then
|
6
|
-
return 1
|
7
|
-
end
|
8
|
-
|
9
|
-
local stored_time = redis.pcall('get', unique_key)
|
10
|
-
if stored_time and stored_time < time then
|
11
|
-
if redis.pcall('set', unique_key, time + expires, 'xx', 'ex', expires) then
|
12
|
-
return 1
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
return 0
|