sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +17 -9
  3. data/.gitignore +1 -3
  4. data/.reek +105 -0
  5. data/.rubocop.yml +36 -1
  6. data/.simplecov +7 -2
  7. data/.travis.yml +11 -10
  8. data/Appraisals +3 -7
  9. data/CHANGELOG.md +17 -0
  10. data/Gemfile +16 -13
  11. data/Guardfile +55 -0
  12. data/README.md +85 -73
  13. data/examples/another_unique_job.rb +13 -0
  14. data/examples/custom_queue_job.rb +12 -0
  15. data/examples/custom_queue_job_with_filter_method.rb +13 -0
  16. data/examples/custom_queue_job_with_filter_proc.rb +16 -0
  17. data/examples/expiring_job.rb +12 -0
  18. data/examples/inline_worker.rb +12 -0
  19. data/examples/just_a_worker.rb +13 -0
  20. data/examples/long_running_job.rb +12 -0
  21. data/examples/main_job.rb +13 -0
  22. data/examples/my_job.rb +12 -0
  23. data/examples/my_unique_job.rb +16 -0
  24. data/examples/my_unique_job_with_filter_method.rb +21 -0
  25. data/examples/my_unique_job_with_filter_proc.rb +19 -0
  26. data/examples/notify_worker.rb +14 -0
  27. data/examples/plain_class.rb +13 -0
  28. data/examples/simple_worker.rb +15 -0
  29. data/examples/spawn_simple_worker.rb +12 -0
  30. data/examples/test_class.rb +9 -0
  31. data/examples/unique_across_workers_job.rb +20 -0
  32. data/examples/unique_job_with_conditional_parameter.rb +18 -0
  33. data/examples/unique_job_with_filter_method.rb +18 -0
  34. data/examples/unique_job_with_nil_unique_args.rb +20 -0
  35. data/examples/unique_job_with_no_unique_args_method.rb +16 -0
  36. data/examples/unique_job_withthout_unique_args_parameter.rb +18 -0
  37. data/examples/unique_on_all_queues_job.rb +16 -0
  38. data/examples/until_and_while_executing_job.rb +13 -0
  39. data/examples/until_executed_2_job.rb +24 -0
  40. data/examples/until_executed_job.rb +25 -0
  41. data/examples/until_executing_job.rb +11 -0
  42. data/examples/until_expired_job.rb +12 -0
  43. data/examples/until_global_expired_job.rb +12 -0
  44. data/examples/while_executing_job.rb +15 -0
  45. data/examples/while_executing_reject_job.rb +14 -0
  46. data/examples/without_argument_job.rb +13 -0
  47. data/lib/sidekiq-unique-jobs.rb +1 -91
  48. data/lib/sidekiq/simulator.rb +15 -16
  49. data/lib/sidekiq_unique_jobs.rb +79 -0
  50. data/lib/sidekiq_unique_jobs/cli.rb +9 -18
  51. data/lib/sidekiq_unique_jobs/client/middleware.rb +16 -18
  52. data/lib/sidekiq_unique_jobs/connection.rb +18 -0
  53. data/lib/sidekiq_unique_jobs/constants.rb +13 -17
  54. data/lib/sidekiq_unique_jobs/core_ext.rb +28 -55
  55. data/lib/sidekiq_unique_jobs/exceptions.rb +30 -0
  56. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +84 -0
  57. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +11 -6
  58. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +6 -58
  59. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +6 -5
  60. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +17 -0
  61. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +20 -34
  62. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +78 -0
  63. data/lib/sidekiq_unique_jobs/lock/while_executing_requeue.rb +20 -0
  64. data/lib/sidekiq_unique_jobs/locksmith.rb +149 -0
  65. data/lib/sidekiq_unique_jobs/logging.rb +30 -0
  66. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +27 -41
  67. data/lib/sidekiq_unique_jobs/scripts.rb +25 -24
  68. data/lib/sidekiq_unique_jobs/server/middleware.rb +12 -20
  69. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +5 -5
  70. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +42 -0
  71. data/lib/sidekiq_unique_jobs/testing.rb +40 -50
  72. data/lib/sidekiq_unique_jobs/timeout.rb +15 -0
  73. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +49 -0
  74. data/lib/sidekiq_unique_jobs/unique_args.rb +33 -77
  75. data/lib/sidekiq_unique_jobs/unlockable.rb +5 -14
  76. data/lib/sidekiq_unique_jobs/util.rb +28 -90
  77. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  78. data/redis/acquire_lock.lua +5 -3
  79. data/redis/create.lua +58 -0
  80. data/redis/delete.lua +11 -0
  81. data/redis/release_stale_locks.lua +90 -0
  82. data/redis/signal.lua +21 -0
  83. data/sidekiq-unique-jobs.gemspec +15 -8
  84. metadata +108 -32
  85. data/lib/sidekiq_unique_jobs/config.rb +0 -17
  86. data/lib/sidekiq_unique_jobs/lock.rb +0 -12
  87. data/lib/sidekiq_unique_jobs/lock/until_timeout.rb +0 -17
  88. data/lib/sidekiq_unique_jobs/run_lock_failed.rb +0 -3
  89. data/lib/sidekiq_unique_jobs/script_mock.rb +0 -66
  90. data/lib/sidekiq_unique_jobs/scripts/acquire_lock.rb +0 -47
  91. data/lib/sidekiq_unique_jobs/scripts/release_lock.rb +0 -49
  92. data/lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb +0 -50
  93. data/lib/sidekiq_unique_jobs/timeout_calculator.rb +0 -67
  94. data/redis/synchronize.lua +0 -16
@@ -1,17 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- class Config < OpenStruct
5
- def inline_testing_enabled?
6
- testing_enabled? && Sidekiq::Testing.inline?
7
- end
8
-
9
- def mocking?
10
- redis_test_mode.to_sym == :mock
11
- end
12
-
13
- def testing_enabled?
14
- Sidekiq.const_defined?(TESTING_CONSTANT, false) && Sidekiq::Testing.enabled?
15
- end
16
- end
17
- end
@@ -1,12 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'sidekiq_unique_jobs/lock/until_executed'
4
- require 'sidekiq_unique_jobs/lock/until_executing'
5
- require 'sidekiq_unique_jobs/lock/while_executing'
6
- require 'sidekiq_unique_jobs/lock/until_timeout'
7
- require 'sidekiq_unique_jobs/lock/until_and_while_executing'
8
-
9
- module SidekiqUniqueJobs
10
- module Lock
11
- end
12
- end
@@ -1,17 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- module Lock
5
- class UntilTimeout < UntilExecuted
6
- def unlock(scope)
7
- return true if scope.to_sym == :server
8
-
9
- raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
10
- end
11
-
12
- def execute(_callback)
13
- yield if block_given?
14
- end
15
- end
16
- end
17
- end
@@ -1,3 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- SidekiqUniqueJobs::RunLockFailed = Class.new(StandardError)
@@ -1,66 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'pathname'
4
- require 'digest/sha1'
5
-
6
- module SidekiqUniqueJobs
7
- module ScriptMock
8
- module_function
9
-
10
- extend SingleForwardable
11
- def_delegator :SidekiqUniqueJobs, :connection
12
-
13
- def call(file_name, redis_pool, options = {})
14
- send(file_name, redis_pool, options)
15
- end
16
-
17
- def acquire_lock(redis_pool, options = {})
18
- connection(redis_pool) do |conn|
19
- unique_key = options[:keys][0]
20
- job_id = options[:argv][0]
21
- expires = options[:argv][1].to_i
22
- stored_jid = conn.get(unique_key)
23
-
24
- return (stored_jid == job_id) ? 1 : 0 if stored_jid
25
- return 0 unless conn.set(unique_key, job_id, nx: true, ex: expires)
26
-
27
- conn.hsetnx(SidekiqUniqueJobs::HASH_KEY, job_id, unique_key)
28
-
29
- return 1
30
- end
31
- end
32
-
33
- def release_lock(redis_pool, options = {})
34
- connection(redis_pool) do |conn|
35
- unique_key = options[:keys][0]
36
- job_id = options[:argv][0]
37
- stored_jid = conn.get(unique_key)
38
-
39
- return -1 unless stored_jid
40
- return 0 unless stored_jid == job_id || stored_jid == '2'
41
-
42
- conn.del(unique_key)
43
- conn.hdel(SidekiqUniqueJobs::HASH_KEY, job_id)
44
-
45
- return 1
46
- end
47
- end
48
-
49
- def synchronize(redis_pool, options = {})
50
- connection(redis_pool) do |conn|
51
- unique_key = options[:keys][0]
52
- time = options[:argv][0].to_i
53
- expires = options[:argv][1].to_f
54
-
55
- return 1 if conn.set(unique_key, time + expires, nx: true, ex: expires)
56
-
57
- stored_time = conn.get(unique_key)
58
- if stored_time && stored_time < time
59
- return 1 if conn.set(unique_key, time + expires, xx: true, ex: expires)
60
- end
61
-
62
- return 0
63
- end
64
- end
65
- end
66
- end
@@ -1,47 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- module Scripts
5
- class AcquireLock
6
- extend Forwardable
7
- def_delegator SidekiqUniqueJobs, :logger
8
-
9
- def self.execute(redis_pool, unique_key, jid, max_lock_time)
10
- new(redis_pool, unique_key, jid, max_lock_time).execute
11
- end
12
-
13
- attr_reader :redis_pool, :unique_key, :jid, :max_lock_time
14
-
15
- def initialize(_redis_pool, unique_key, jid, max_lock_time)
16
- raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
17
- raise JidMissing, 'jid is required' if jid.nil?
18
- raise MaxLockTimeMissing, 'max_lock_time is required' if max_lock_time.nil?
19
-
20
- @unique_key = unique_key
21
- @jid = jid
22
- @max_lock_time = max_lock_time
23
- end
24
-
25
- def execute
26
- result = Scripts.call(:acquire_lock, redis_pool,
27
- keys: [unique_key],
28
- argv: [jid, max_lock_time])
29
-
30
- handle_result(result)
31
- end
32
-
33
- def handle_result(result)
34
- case result
35
- when 1
36
- logger.debug { "successfully acquired lock #{unique_key} for #{max_lock_time} seconds" }
37
- true
38
- when 0
39
- logger.debug { "failed to acquire lock for #{unique_key}" }
40
- false
41
- else
42
- raise UnexpectedValue, "failed to acquire lock : unexpected return value (#{result})"
43
- end
44
- end
45
- end
46
- end
47
- end
@@ -1,49 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- module Scripts
5
- class ReleaseLock
6
- extend Forwardable
7
- def_delegator SidekiqUniqueJobs, :logger
8
-
9
- def self.execute(redis_pool, unique_key, jid)
10
- new(redis_pool, unique_key, jid).execute
11
- end
12
-
13
- attr_reader :redis_pool, :unique_key, :jid
14
-
15
- def initialize(redis_pool, unique_key, jid)
16
- raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
17
- raise JidMissing, 'jid is required' if jid.nil?
18
-
19
- @redis_pool = redis_pool
20
- @unique_key = unique_key
21
- @jid = jid
22
- end
23
-
24
- def execute
25
- result = Scripts.call(:release_lock, redis_pool,
26
- keys: [unique_key],
27
- argv: [jid])
28
-
29
- handle_result(result)
30
- end
31
-
32
- def handle_result(result)
33
- case result
34
- when 1
35
- logger.debug { "successfully unlocked #{unique_key}" }
36
- true
37
- when 0
38
- logger.debug { "expiring lock #{unique_key} is not owned by #{jid}" }
39
- false
40
- when -1
41
- logger.debug { "#{unique_key} is not a known key" }
42
- false
43
- else
44
- raise UnexpectedValue, "failed to release lock : unexpected return value (#{result})"
45
- end
46
- end
47
- end
48
- end
49
- end
@@ -1,50 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'sidekiq/testing'
4
-
5
- module Sidekiq
6
- module Worker
7
- module ClassMethods
8
- # Clear all jobs for this worker
9
- def clear
10
- jobs.each do |job|
11
- unlock(job) if Sidekiq::Testing.fake?
12
- end
13
-
14
- Sidekiq::Queues[queue].clear
15
- jobs.clear
16
- end
17
-
18
- unless respond_to?(:execute_job)
19
- def execute_job(worker, args)
20
- worker.perform(*args)
21
- end
22
- end
23
-
24
- def unlock(job)
25
- SidekiqUniqueJobs::Unlockable.unlock(job)
26
- end
27
- end
28
-
29
- module Overrides
30
- def self.included(base)
31
- base.extend Testing
32
- base.class_eval do
33
- class << self
34
- alias_method :clear_all_orig, :clear_all
35
- alias_method :clear_all, :clear_all_ext
36
- end
37
- end
38
- end
39
-
40
- module Testing
41
- def clear_all_ext
42
- SidekiqUniqueJobs::Util.del('*', 1000, false) unless SidekiqUniqueJobs.mocked?
43
- clear_all_orig
44
- end
45
- end
46
- end
47
-
48
- include Overrides
49
- end
50
- end
@@ -1,67 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- class TimeoutCalculator
5
- def self.for_item(item)
6
- new(item)
7
- end
8
-
9
- def initialize(item)
10
- @item = item
11
- end
12
-
13
- def time_until_scheduled
14
- scheduled = item[AT_KEY]
15
- return 0 unless scheduled
16
- (Time.at(scheduled) - Time.now.utc).to_i
17
- end
18
-
19
- def seconds
20
- raise NotImplementedError
21
- end
22
-
23
- def worker_class_queue_lock_expiration
24
- worker_class_expiration_for QUEUE_LOCK_TIMEOUT_KEY
25
- end
26
-
27
- def worker_class_run_lock_expiration
28
- worker_class_expiration_for RUN_LOCK_TIMEOUT_KEY
29
- end
30
-
31
- def worker_class
32
- @worker_class ||= SidekiqUniqueJobs.worker_class_constantize(item[CLASS_KEY])
33
- end
34
-
35
- private
36
-
37
- def worker_class_expiration_for(key)
38
- return unless worker_class.respond_to?(:get_sidekiq_options)
39
- worker_class.get_sidekiq_options[key]
40
- end
41
-
42
- attr_reader :item
43
- end
44
-
45
- class RunLockTimeoutCalculator < TimeoutCalculator
46
- def seconds
47
- @seconds ||= (
48
- worker_class_run_lock_expiration ||
49
- SidekiqUniqueJobs.config.default_run_lock_expiration
50
- ).to_i
51
- end
52
- end
53
-
54
- class QueueLockTimeoutCalculator < TimeoutCalculator
55
- def seconds
56
- queue_lock_expiration + time_until_scheduled
57
- end
58
-
59
- def queue_lock_expiration
60
- @queue_lock_expiration ||=
61
- (
62
- worker_class_queue_lock_expiration ||
63
- SidekiqUniqueJobs.config.default_queue_lock_expiration
64
- ).to_i
65
- end
66
- end
67
- end
@@ -1,16 +0,0 @@
1
- local unique_key = KEYS[1]
2
- local time = ARGV[1]
3
- local expires = ARGV[2]
4
-
5
- if redis.pcall('set', unique_key, time + expires, 'nx', 'ex', expires) then
6
- return 1
7
- end
8
-
9
- local stored_time = redis.pcall('get', unique_key)
10
- if stored_time and stored_time < time then
11
- if redis.pcall('set', unique_key, time + expires, 'xx', 'ex', expires) then
12
- return 1
13
- end
14
- end
15
-
16
- return 0