sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +155 -20
  3. data/README.md +349 -112
  4. data/lib/sidekiq-unique-jobs.rb +2 -0
  5. data/lib/sidekiq_unique_jobs.rb +43 -6
  6. data/lib/sidekiq_unique_jobs/batch_delete.rb +121 -0
  7. data/lib/sidekiq_unique_jobs/changelog.rb +71 -0
  8. data/lib/sidekiq_unique_jobs/cli.rb +20 -29
  9. data/lib/sidekiq_unique_jobs/config.rb +193 -0
  10. data/lib/sidekiq_unique_jobs/connection.rb +5 -4
  11. data/lib/sidekiq_unique_jobs/constants.rb +36 -24
  12. data/lib/sidekiq_unique_jobs/core_ext.rb +38 -0
  13. data/lib/sidekiq_unique_jobs/digests.rb +78 -93
  14. data/lib/sidekiq_unique_jobs/exceptions.rb +152 -8
  15. data/lib/sidekiq_unique_jobs/job.rb +3 -3
  16. data/lib/sidekiq_unique_jobs/json.rb +34 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +93 -0
  18. data/lib/sidekiq_unique_jobs/lock.rb +295 -0
  19. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +49 -43
  20. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  21. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  22. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +8 -17
  23. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +5 -5
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +1 -23
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +65 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +12 -8
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +1 -1
  28. data/lib/sidekiq_unique_jobs/lock_config.rb +95 -0
  29. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  30. data/lib/sidekiq_unique_jobs/locksmith.rb +255 -99
  31. data/lib/sidekiq_unique_jobs/logging.rb +148 -22
  32. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  33. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  34. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +46 -0
  35. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  36. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  37. data/lib/sidekiq_unique_jobs/lua/find_digest_in_sorted_set.lua +24 -0
  38. data/lib/sidekiq_unique_jobs/lua/lock.lua +91 -0
  39. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  40. data/lib/sidekiq_unique_jobs/lua/queue.lua +83 -0
  41. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +86 -0
  42. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  43. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  44. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +19 -0
  45. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  46. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +46 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/find_digest_in_sorted_set.lua +24 -0
  51. data/lib/sidekiq_unique_jobs/lua/unlock.lua +99 -0
  52. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  53. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  54. data/lib/sidekiq_unique_jobs/middleware.rb +62 -31
  55. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  56. data/lib/sidekiq_unique_jobs/middleware/server.rb +27 -0
  57. data/lib/sidekiq_unique_jobs/normalizer.rb +3 -3
  58. data/lib/sidekiq_unique_jobs/on_conflict.rb +22 -9
  59. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +8 -4
  60. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +59 -13
  61. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +42 -13
  62. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +4 -4
  63. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +24 -5
  64. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +47 -23
  65. data/lib/sidekiq_unique_jobs/orphans/manager.rb +100 -0
  66. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  67. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +201 -0
  68. data/lib/sidekiq_unique_jobs/profiler.rb +51 -0
  69. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  70. data/lib/sidekiq_unique_jobs/redis/entity.rb +94 -0
  71. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  72. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  73. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  74. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +59 -0
  75. data/lib/sidekiq_unique_jobs/redis/string.rb +49 -0
  76. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +19 -0
  77. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +43 -0
  78. data/lib/sidekiq_unique_jobs/{scripts.rb → script.rb} +43 -29
  79. data/lib/sidekiq_unique_jobs/script/caller.rb +125 -0
  80. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  81. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +92 -65
  82. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +166 -28
  83. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +10 -11
  84. data/lib/sidekiq_unique_jobs/testing.rb +47 -15
  85. data/lib/sidekiq_unique_jobs/time_calculator.rb +103 -0
  86. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  87. data/lib/sidekiq_unique_jobs/unique_args.rb +19 -21
  88. data/lib/sidekiq_unique_jobs/unlockable.rb +11 -2
  89. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  90. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +151 -0
  91. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  92. data/lib/sidekiq_unique_jobs/version_check.rb +1 -1
  93. data/lib/sidekiq_unique_jobs/web.rb +25 -19
  94. data/lib/sidekiq_unique_jobs/web/helpers.rb +98 -6
  95. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  96. data/lib/sidekiq_unique_jobs/web/views/locks.erb +52 -0
  97. data/lib/tasks/changelog.rake +4 -3
  98. metadata +70 -35
  99. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  100. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  101. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  102. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  103. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  104. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  105. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  106. data/redis/acquire_lock.lua +0 -21
  107. data/redis/convert_legacy_lock.lua +0 -13
  108. data/redis/delete.lua +0 -14
  109. data/redis/delete_by_digest.lua +0 -23
  110. data/redis/delete_job_by_digest.lua +0 -60
  111. data/redis/lock.lua +0 -62
  112. data/redis/release_stale_locks.lua +0 -90
  113. data/redis/unlock.lua +0 -35
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq client process
7
+ #
8
+ # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ #
10
+ class ClientValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq client process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_client_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
24
+ lock_config
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq server process
7
+ #
8
+ # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ #
10
+ class ServerValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:replace].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq server process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_server_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
24
+ end
25
+ end
26
+ end
27
+ end
@@ -17,30 +17,21 @@ module SidekiqUniqueJobs
17
17
  # @yield to the worker class perform method
18
18
  def execute
19
19
  if unlock
20
- lock_on_failure do
21
- runtime_lock.execute { return yield }
22
- end
20
+ runtime_lock.execute { return yield }
23
21
  else
24
- log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST_KEY]} #{item[JID_KEY]}"
22
+ log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST]} #{item[JID]}"
25
23
  end
26
- ensure
27
- runtime_lock.delete!
28
24
  end
29
25
 
26
+ #
27
+ # Lock only when the server is processing the job
28
+ #
29
+ #
30
+ # @return [SidekiqUniqueJobs::Lock::WhileExecuting] an instance of a lock
31
+ #
30
32
  def runtime_lock
31
33
  @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
32
34
  end
33
-
34
- private
35
-
36
- def lock_on_failure
37
- yield
38
- runtime_lock.delete!
39
- rescue Exception # rubocop:disable Lint/RescueException
40
- log_error("Failed to execute job, restoring lock")
41
- lock
42
- raise
43
- end
44
35
  end
45
36
  end
46
37
  end
@@ -13,11 +13,11 @@ module SidekiqUniqueJobs
13
13
  # Executes in the Sidekiq server process
14
14
  # @yield to the worker class perform method
15
15
  def execute
16
- if locked?
17
- with_cleanup { yield }
18
- else
19
- log_warn "the unique_key: #{item[UNIQUE_DIGEST_KEY]} is not locked, allowing job to silently complete"
20
- nil
16
+ lock do
17
+ yield
18
+ unlock_with_callback
19
+ callback_safely
20
+ item[JID]
21
21
  end
22
22
  end
23
23
  end
@@ -2,29 +2,7 @@
2
2
 
3
3
  module SidekiqUniqueJobs
4
4
  class Lock
5
- # Locks jobs until the lock has expired
6
- # - Locks on perform_in or perform_async
7
- # - Unlocks when the expiration is hit
8
- #
9
- # See {#lock} for more information about the client.
10
- # See {#execute} for more information about the server
11
- #
12
- # @author Mikael Henriksson <mikael@zoolutions.se>
13
- class UntilExpired < BaseLock
14
- # Prevents these locks from being unlocked
15
- # @return [true] always returns true
16
- def unlock
17
- true
18
- end
19
-
20
- # Executes in the Sidekiq server process
21
- # @yield to the worker class perform method
22
- def execute
23
- return unless locked?
24
-
25
- yield
26
- # this lock does not handle after_unlock since we don't know when that would happen
27
- end
5
+ class UntilExpired < UntilExecuted
28
6
  end
29
7
  end
30
8
  end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validator base class to avoid some duplication
7
+ #
8
+ # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ #
10
+ class Validator
11
+ #
12
+ # Shorthand for `new(options).validate`
13
+ #
14
+ # @param [Hash] options the sidekiq_options for the worker being validated
15
+ #
16
+ # @return [LockConfig] the lock configuration with errors if any
17
+ #
18
+ def self.validate(options)
19
+ new(options).validate
20
+ end
21
+
22
+ #
23
+ # @!attribute [r] lock_config
24
+ # @return [LockConfig] the lock configuration for this worker
25
+ attr_reader :lock_config
26
+
27
+ #
28
+ # Initialize a new validator
29
+ #
30
+ # @param [Hash] options the sidekiq_options for the worker being validated
31
+ #
32
+ def initialize(options)
33
+ @lock_config = LockConfig.new(options)
34
+ end
35
+
36
+ #
37
+ # Validate the workers lock configuration
38
+ #
39
+ #
40
+ # @return [LockConfig] the lock configuration with errors if any
41
+ #
42
+ def validate
43
+ case lock_config.type
44
+ when :while_executing
45
+ validate_server
46
+ when :until_executing
47
+ validate_client
48
+ else
49
+ validate_client
50
+ validate_server
51
+ end
52
+
53
+ lock_config
54
+ end
55
+
56
+ def validate_client
57
+ ClientValidator.validate(lock_config)
58
+ end
59
+
60
+ def validate_server
61
+ ServerValidator.validate(lock_config)
62
+ end
63
+ end
64
+ end
65
+ end
@@ -13,6 +13,9 @@ module SidekiqUniqueJobs
13
13
  class WhileExecuting < BaseLock
14
14
  RUN_SUFFIX ||= ":RUN"
15
15
 
16
+ include SidekiqUniqueJobs::OptionsWithFallback
17
+ include SidekiqUniqueJobs::Logging::Middleware
18
+
16
19
  # @param [Hash] item the Sidekiq job hash
17
20
  # @param [Proc] callback callback to call after unlock
18
21
  # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -33,13 +36,14 @@ module SidekiqUniqueJobs
33
36
  # These jobs are locked in the server process not from the client
34
37
  # @yield to the worker class perform method
35
38
  def execute
36
- return strategy&.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
37
-
38
- yield
39
- unlock_with_callback
40
- rescue Exception # rubocop:disable Lint/RescueException
41
- delete!
42
- raise
39
+ with_logging_context do
40
+ return strategy.call unless locksmith.lock do
41
+ yield
42
+ callback_safely
43
+ end
44
+ end
45
+ ensure
46
+ locksmith.unlock
43
47
  end
44
48
 
45
49
  private
@@ -47,7 +51,7 @@ module SidekiqUniqueJobs
47
51
  # This is safe as the base_lock always creates a new digest
48
52
  # The append there for needs to be done every time
49
53
  def append_unique_key_suffix
50
- item[UNIQUE_DIGEST_KEY] = item[UNIQUE_DIGEST_KEY] + RUN_SUFFIX
54
+ item[UNIQUE_DIGEST] = item[UNIQUE_DIGEST] + RUN_SUFFIX
51
55
  end
52
56
  end
53
57
  end
@@ -14,7 +14,7 @@ module SidekiqUniqueJobs
14
14
  # Overridden with a forced {OnConflict::Reject} strategy
15
15
  # @return [OnConflict::Reject] a reject strategy
16
16
  def strategy
17
- @strategy ||= OnConflict.find_strategy(:reject).new(item)
17
+ @strategy ||= OnConflict.find_strategy(:reject).new(item, redis_pool)
18
18
  end
19
19
  end
20
20
  end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Gathers all configuration for a lock
6
+ # which helps reduce the amount of instance variables
7
+ #
8
+ # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ #
10
+ class LockConfig
11
+ #
12
+ # @!attribute [r] type
13
+ # @return [Symbol] the type of lock
14
+ attr_reader :type
15
+ #
16
+ # @!attribute [r] worker
17
+ # @return [Symbol] the worker class
18
+ attr_reader :worker
19
+ #
20
+ # @!attribute [r] limit
21
+ # @return [Integer] the number of simultaneous locks
22
+ attr_reader :limit
23
+ #
24
+ # @!attribute [r] timeout
25
+ # @return [Integer, nil] the time to wait for a lock
26
+ attr_reader :timeout
27
+ #
28
+ # @!attribute [r] ttl
29
+ # @return [Integer, nil] the time (in seconds) to live after successful
30
+ attr_reader :ttl
31
+ #
32
+ # @!attribute [r] ttl
33
+ # @return [Integer, nil] the time (in milliseconds) to live after successful
34
+ attr_reader :pttl
35
+ #
36
+ # @!attribute [r] lock_info
37
+ # @return [Boolean] indicate wether to use lock_info or not
38
+ attr_reader :lock_info
39
+ #
40
+ # @!attribute [r] on_conflict
41
+ # @return [Symbol, Hash<Symbol, Symbol>] the strategies to use as conflict resolution
42
+ attr_reader :on_conflict
43
+ #
44
+ # @!attribute [r] errors
45
+ # @return [Array<Hash<Symbol, Array<String>] a collection of configuration errors
46
+ attr_reader :errors
47
+
48
+ def self.from_worker(options)
49
+ new(options.stringify_keys)
50
+ end
51
+
52
+ def initialize(job_hash = {})
53
+ @type = job_hash[LOCK]&.to_sym
54
+ @worker = job_hash[CLASS]
55
+ @limit = job_hash.fetch(LOCK_LIMIT) { 1 }
56
+ @timeout = job_hash.fetch(LOCK_TIMEOUT) { 0 }
57
+ @ttl = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION) { nil } }.to_i
58
+ @pttl = ttl * 1_000
59
+ @lock_info = job_hash.fetch(LOCK_INFO) { SidekiqUniqueJobs.config.lock_info }
60
+ @on_conflict = job_hash.fetch(ON_CONFLICT) { nil }
61
+ @errors = job_hash.fetch(ERRORS) { {} }
62
+
63
+ @on_client_conflict = job_hash[ON_CLIENT_CONFLICT]
64
+ @on_server_conflict = job_hash[ON_SERVER_CONFLICT]
65
+ end
66
+
67
+ def wait_for_lock?
68
+ timeout.nil? || timeout.positive?
69
+ end
70
+
71
+ def valid?
72
+ errors.empty?
73
+ end
74
+
75
+ def errors_as_string
76
+ @errors_as_string ||= begin
77
+ error_msg = +"\t"
78
+ error_msg << lock_config.errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")
79
+ error_msg
80
+ end
81
+ end
82
+
83
+ # the strategy to use as conflict resolution from sidekiq client
84
+ def on_client_conflict
85
+ @on_client_conflict ||= on_conflict&.(:[], :client) if on_conflict.is_a?(Hash)
86
+ @on_client_conflict ||= on_conflict
87
+ end
88
+
89
+ # the strategy to use as conflict resolution from sidekiq server
90
+ def on_server_conflict
91
+ @on_client_conflict ||= on_conflict&.(:[], :server) if on_conflict.is_a?(Hash)
92
+ @on_server_conflict ||= on_conflict
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class Info provides information about a lock
6
+ #
7
+ # @author Mikael Henriksson <mikael@zoolutions.se>
8
+ #
9
+ class LockInfo < Redis::String
10
+ #
11
+ # Returns the value for this key as a hash
12
+ #
13
+ #
14
+ # @return [Hash]
15
+ #
16
+ def value
17
+ @value ||= load_json(super)
18
+ end
19
+
20
+ #
21
+ # Check if this redis string is blank
22
+ #
23
+ #
24
+ # @return [Boolean]
25
+ #
26
+ def none?
27
+ value.nil? || value.empty?
28
+ end
29
+
30
+ #
31
+ # Check if this redis string has a value
32
+ #
33
+ #
34
+ # @return [Boolean]
35
+ #
36
+ def present?
37
+ !none?
38
+ end
39
+
40
+ #
41
+ # Quick access to the hash members for the value
42
+ #
43
+ # @param [String, Symbol] key the key who's value to retrieve
44
+ #
45
+ # @return [Object]
46
+ #
47
+ def [](key)
48
+ value[key.to_s] if value.is_a?(Hash)
49
+ end
50
+
51
+ #
52
+ # Writes the lock info to redis
53
+ #
54
+ # @param [Hash] obj the information to store at key
55
+ #
56
+ # @return [Hash]
57
+ #
58
+ def set(obj)
59
+ return unless SidekiqUniqueJobs.config.lock_info
60
+ raise InvalidArgument, "argument `obj` (#{obj}) needs to be a hash" unless obj.is_a?(Hash)
61
+
62
+ json = dump_json(obj)
63
+ @value = load_json(json)
64
+ super(json)
65
+ value
66
+ end
67
+ end
68
+ end
@@ -4,63 +4,94 @@ module SidekiqUniqueJobs
4
4
  # Lock manager class that handles all the various locks
5
5
  #
6
6
  # @author Mikael Henriksson <mikael@zoolutions.se>
7
- # rubocop:disable Metrics/ClassLength
8
- class Locksmith
7
+ class Locksmith # rubocop:disable Metrics/ClassLength
8
+ # includes "SidekiqUniqueJobs::Connection"
9
+ # @!parse include SidekiqUniqueJobs::Connection
9
10
  include SidekiqUniqueJobs::Connection
10
11
 
12
+ # includes "SidekiqUniqueJobs::Logging"
13
+ # @!parse include SidekiqUniqueJobs::Logging
14
+ include SidekiqUniqueJobs::Logging
15
+
16
+ # includes "SidekiqUniqueJobs::Timing"
17
+ # @!parse include SidekiqUniqueJobs::Timing
18
+ include SidekiqUniqueJobs::Timing
19
+
20
+ # includes "SidekiqUniqueJobs::Script::Caller"
21
+ # @!parse include SidekiqUniqueJobs::Script::Caller
22
+ include SidekiqUniqueJobs::Script::Caller
23
+
24
+ # includes "SidekiqUniqueJobs::JSON"
25
+ # @!parse include SidekiqUniqueJobs::JSON
26
+ include SidekiqUniqueJobs::JSON
27
+
28
+ CLOCK_DRIFT_FACTOR = 0.01
29
+
30
+ #
31
+ # @!attribute [r] key
32
+ # @return [Key] the key used for locking
33
+ attr_reader :key
34
+ #
35
+ # @!attribute [r] job_id
36
+ # @return [String] a sidekiq JID
37
+ attr_reader :job_id
38
+ #
39
+ # @!attribute [r] config
40
+ # @return [LockConfig] the configuration for this lock
41
+ attr_reader :config
42
+ #
43
+ # @!attribute [r] item
44
+ # @return [Hash] a sidekiq job hash
45
+ attr_reader :item
46
+
47
+ #
48
+ # Initialize a new Locksmith instance
49
+ #
11
50
  # @param [Hash] item a Sidekiq job hash
12
- # @option item [Integer] :lock_expiration the configured expiration
51
+ # @option item [Integer] :lock_ttl the configured expiration
13
52
  # @option item [String] :jid the sidekiq job id
14
53
  # @option item [String] :unique_digest the unique digest (See: {UniqueArgs#unique_digest})
15
54
  # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
55
+ #
16
56
  def initialize(item, redis_pool = nil)
17
- # @concurrency = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
18
- @ttl = item[LOCK_EXPIRATION_KEY] || item[LOCK_TTL_KEY]
19
- @jid = item[JID_KEY]
20
- @unique_digest = item[UNIQUE_DIGEST_KEY] || item[LOCK_DIGEST_KEY]
21
- @lock_type = item[LOCK_KEY] || item[UNIQUE_KEY]
22
- @lock_type &&= @lock_type.to_sym
23
- @redis_pool = redis_pool
57
+ @item = item
58
+ @key = Key.new(item[UNIQUE_DIGEST])
59
+ @job_id = item[JID]
60
+ @config = LockConfig.new(item)
61
+ @redis_pool = redis_pool
24
62
  end
25
63
 
26
64
  #
27
- # Deletes the lock unless it has a ttl set
65
+ # Deletes the lock unless it has a pttl set
28
66
  #
29
67
  #
30
68
  def delete
31
- return if ttl
69
+ return if config.pttl.positive?
32
70
 
33
71
  delete!
34
72
  end
35
73
 
36
- # Deletes the lock regardless of if it has a ttl set
74
+ #
75
+ # Deletes the lock regardless of if it has a pttl set
76
+ #
37
77
  def delete!
38
- Scripts.call(
39
- :delete,
40
- redis_pool,
41
- keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
42
- )
78
+ call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
43
79
  end
44
80
 
45
81
  #
46
- # Create a lock for the item
47
- #
48
- # @param [Integer] timeout the number of seconds to wait for a lock.
82
+ # Create a lock for the Sidekiq job
49
83
  #
50
- # @return [String] the Sidekiq job_id (jid)
84
+ # @return [String] the Sidekiq job_id that was locked/queued
51
85
  #
52
- #
53
- def lock(timeout = nil, &block)
54
- Scripts.call(:lock, redis_pool,
55
- keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest],
56
- argv: [jid, ttl, lock_type])
86
+ def lock(&block)
87
+ redis(redis_pool) do |conn|
88
+ return lock_async(conn, &block) if block_given?
57
89
 
58
- grab_token(timeout) do |token|
59
- touch_grabbed_token(token)
60
- return_token_or_block_value(token, &block)
90
+ lock_sync(conn) do
91
+ return job_id
92
+ end
61
93
  end
62
94
  end
63
- alias wait lock
64
95
 
65
96
  #
66
97
  # Removes the lock keys from Redis if locked by the provided jid/token
@@ -68,122 +99,247 @@ module SidekiqUniqueJobs
68
99
  # @return [false] unless locked?
69
100
  # @return [String] Sidekiq job_id (jid) if successful
70
101
  #
71
- def unlock(token = nil)
72
- token ||= jid
73
- return false unless locked?(token)
102
+ def unlock(conn = nil)
103
+ return false unless locked?(conn)
74
104
 
75
- unlock!(token)
105
+ unlock!(conn)
76
106
  end
77
107
 
78
108
  #
79
109
  # Removes the lock keys from Redis
80
110
  #
81
- # @param [String] token the token to unlock (defaults to jid)
82
- #
83
111
  # @return [false] unless locked?
84
112
  # @return [String] Sidekiq job_id (jid) if successful
85
113
  #
86
- def unlock!(token = nil)
87
- token ||= jid
114
+ def unlock!(conn = nil)
115
+ call_script(:unlock, key.to_a, argv, conn)
116
+ end
88
117
 
89
- Scripts.call(
90
- :unlock,
91
- redis_pool,
92
- keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
93
- argv: [token, ttl, lock_type],
94
- )
118
+ # Checks if this instance is considered locked
119
+ #
120
+ # @return [true, false] true when the :LOCKED hash contains the job_id
121
+ #
122
+ def locked?(conn = nil)
123
+ return taken?(conn) if conn
124
+
125
+ redis { |rcon| taken?(rcon) }
95
126
  end
96
127
 
97
128
  #
98
- # @param [String] token the unique token to check for a lock.
99
- # nil will default to the jid provided in the initializer
100
- # @return [true, false]
129
+ # Nicely formatted string with information about self
101
130
  #
102
- # Checks if this instance is considered locked
103
131
  #
104
- # @param [<type>] token <description>
132
+ # @return [String]
105
133
  #
106
- # @return [<type>] <description>
134
+ def to_s
135
+ "Locksmith##{object_id}(digest=#{key} job_id=#{job_id}, locked=#{locked?})"
136
+ end
137
+
138
+ #
139
+ # @see to_s
107
140
  #
108
- def locked?(token = nil)
109
- token ||= jid
141
+ def inspect
142
+ to_s
143
+ end
110
144
 
111
- convert_legacy_lock(token)
112
- redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
145
+ #
146
+ # Compare this locksmith with another
147
+ #
148
+ # @param [Locksmith] other the locksmith to compare with
149
+ #
150
+ # @return [true, false]
151
+ #
152
+ def ==(other)
153
+ key == other.key && job_id == other.job_id
113
154
  end
114
155
 
115
156
  private
116
157
 
117
- attr_reader :unique_digest, :ttl, :jid, :redis_pool, :lock_type
158
+ attr_reader :redis_pool
118
159
 
119
- def convert_legacy_lock(token)
120
- Scripts.call(
121
- :convert_legacy_lock,
122
- redis_pool,
123
- keys: [grabbed_key, unique_digest],
124
- argv: [token, current_time.to_f],
125
- )
160
+ def argv
161
+ [job_id, config.pttl, config.type, config.limit]
126
162
  end
127
163
 
128
- def grab_token(timeout = nil)
129
- redis(redis_pool) do |conn|
130
- if timeout.nil? || timeout.positive?
131
- # passing timeout 0 to blpop causes it to block
132
- _key, token = conn.blpop(available_key, timeout || 0)
133
- else
134
- token = conn.lpop(available_key)
135
- end
164
+ #
165
+ # Used for runtime locks that need automatic unlock after yielding
166
+ #
167
+ # @param [Redis] conn a redis connection
168
+ #
169
+ # @return [nil] when lock was not possible
170
+ # @return [Object] whatever the block returns when lock was acquired
171
+ #
172
+ # @yieldparam [String] job_id a Sidekiq JID
173
+ #
174
+ def lock_async(conn)
175
+ return yield job_id if locked?(conn)
136
176
 
137
- return yield jid if token
177
+ enqueue(conn) do
178
+ primed_async(conn) do
179
+ locked_token = call_script(:lock, key.to_a, argv, conn)
180
+ return yield job_id if locked_token == job_id
181
+ end
138
182
  end
183
+ ensure
184
+ unlock!(conn)
139
185
  end
140
186
 
141
- def touch_grabbed_token(token)
142
- redis(redis_pool) do |conn|
143
- conn.hset(grabbed_key, token, current_time.to_f)
144
- conn.expire(grabbed_key, ttl) if ttl && lock_type == :until_expired
187
+ #
188
+ # Pops an enqueued token
189
+ # @note Used for runtime locks to avoid problems with blocking commands
190
+ # in current thread
191
+ #
192
+ # @param [Redis] conn a redis connection
193
+ #
194
+ # @return [nil] when lock was not possible
195
+ # @return [Object] whatever the block returns when lock was acquired
196
+ #
197
+ def primed_async(conn)
198
+ return yield if Concurrent::Promises.future(conn) { |red_con| pop_queued(red_con) }.value
199
+
200
+ warn_about_timeout
201
+ end
202
+
203
+ #
204
+ # Used for non-runtime locks (no block was given)
205
+ #
206
+ # @param [Redis] conn a redis connection
207
+ #
208
+ # @return [nil] when lock was not possible
209
+ # @return [Object] whatever the block returns when lock was acquired
210
+ #
211
+ # @yieldparam [String] job_id a Sidekiq JID
212
+ #
213
+ def lock_sync(conn)
214
+ return yield job_id if locked?(conn)
215
+
216
+ enqueue(conn) do
217
+ primed_sync(conn) do
218
+ locked_token = call_script(:lock, key.to_a, argv, conn)
219
+ return yield job_id if locked_token == job_id
220
+ end
145
221
  end
146
222
  end
147
223
 
148
- def return_token_or_block_value(token)
149
- return token unless block_given?
224
+ #
225
+ # Pops an enqueued token
226
+ # @note Used for non-runtime locks
227
+ #
228
+ # @param [Redis] conn a redis connection
229
+ #
230
+ # @return [nil] when lock was not possible
231
+ # @return [Object] whatever the block returns when lock was acquired
232
+ #
233
+ def primed_sync(conn)
234
+ return yield if pop_queued(conn)
235
+
236
+ warn_about_timeout
237
+ end
150
238
 
151
- # The reason for begin is to only signal when we have a block
152
- begin
153
- yield token
154
- ensure
155
- unlock(token)
239
+ #
240
+ # Does the actual popping of the enqueued token
241
+ #
242
+ # @param [Redis] conn a redis connection
243
+ #
244
+ # @return [String] a previously enqueued token (now taken off the queue)
245
+ #
246
+ def pop_queued(conn)
247
+ if config.wait_for_lock?
248
+ brpoplpush(conn)
249
+ else
250
+ rpoplpush(conn)
156
251
  end
157
252
  end
158
253
 
159
- def available_key
160
- @available_key ||= namespaced_key("AVAILABLE")
254
+ #
255
+ # @api private
256
+ #
257
+ def brpoplpush(conn)
258
+ # passing timeout 0 to brpoplpush causes it to block indefinitely
259
+ conn.brpoplpush(key.queued, key.primed, timeout: config.timeout || 0)
260
+ end
261
+
262
+ #
263
+ # @api private
264
+ #
265
+ def rpoplpush(conn)
266
+ conn.rpoplpush(key.queued, key.primed)
161
267
  end
162
268
 
163
- def exists_key
164
- @exists_key ||= namespaced_key("EXISTS")
269
+ #
270
+ # Prepares all the various lock data
271
+ #
272
+ # @param [Redis] conn a redis connection
273
+ #
274
+ # @return [nil] when redis was already prepared for this lock
275
+ # @return [yield<String>] when successfully enqueued
276
+ #
277
+ def enqueue(conn)
278
+ queued_token, elapsed = timed do
279
+ call_script(:queue, key.to_a, argv, conn)
280
+ end
281
+
282
+ validity = config.pttl - elapsed - drift(config.pttl)
283
+
284
+ return unless queued_token && (validity >= 0 || config.pttl.zero?)
285
+
286
+ write_lock_info(conn)
287
+ yield queued_token
165
288
  end
166
289
 
167
- def grabbed_key
168
- @grabbed_key ||= namespaced_key("GRABBED")
290
+ #
291
+ # Writes lock information to redis.
292
+ # The lock information contains information about worker, queue, limit etc.
293
+ #
294
+ #
295
+ # @return [void]
296
+ #
297
+ def write_lock_info(conn)
298
+ return unless config.lock_info
299
+
300
+ conn.set(key.info, lock_info)
169
301
  end
170
302
 
171
- def version_key
172
- @version_key ||= namespaced_key("VERSION")
303
+ #
304
+ # Used to combat redis imprecision with ttl/pttl
305
+ #
306
+ # @param [Integer] val the value to compute drift for
307
+ #
308
+ # @return [Integer] a computed drift value
309
+ #
310
+ def drift(val)
311
+ # Add 2 milliseconds to the drift to account for Redis expires
312
+ # precision, which is 1 millisecond, plus 1 millisecond min drift
313
+ # for small TTLs.
314
+ (val.to_i * CLOCK_DRIFT_FACTOR).to_i + 2
173
315
  end
174
316
 
175
- def namespaced_key(variable)
176
- "#{unique_digest}:#{variable}"
317
+ #
318
+ # Checks if the lock has been taken
319
+ #
320
+ # @param [Redis] conn a redis connection
321
+ #
322
+ # @return [true, false]
323
+ #
324
+ def taken?(conn)
325
+ conn.hexists(key.locked, job_id)
177
326
  end
178
327
 
179
- def current_time
180
- seconds, microseconds_with_frac = redis_time
181
- Time.at(seconds, microseconds_with_frac)
328
+ def warn_about_timeout
329
+ log_warn("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
182
330
  end
183
331
 
184
- def redis_time
185
- redis(&:time)
332
+ def lock_info
333
+ @lock_info ||= dump_json(
334
+ WORKER => item[CLASS],
335
+ QUEUE => item[QUEUE],
336
+ LIMIT => item[LOCK_LIMIT],
337
+ TIMEOUT => item[LOCK_TIMEOUT],
338
+ TTL => item[LOCK_TTL],
339
+ LOCK => config.type,
340
+ UNIQUE_ARGS => item[UNIQUE_ARGS],
341
+ TIME => now_f,
342
+ )
186
343
  end
187
344
  end
188
- # rubocop:enable Metrics/ClassLength
189
345
  end