sidekiq-unique-jobs 6.0.23 → 7.1.12

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (127) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +922 -41
  3. data/README.md +821 -284
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +123 -0
  5. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  6. data/lib/sidekiq_unique_jobs/cli.rb +34 -31
  7. data/lib/sidekiq_unique_jobs/config.rb +314 -0
  8. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  9. data/lib/sidekiq_unique_jobs/constants.rb +45 -24
  10. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  11. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +70 -102
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  14. data/lib/sidekiq_unique_jobs/job.rb +41 -12
  15. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  16. data/lib/sidekiq_unique_jobs/key.rb +93 -0
  17. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  18. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  19. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  20. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +40 -15
  21. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +25 -7
  22. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +22 -2
  23. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +26 -16
  24. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  25. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +23 -12
  26. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  27. data/lib/sidekiq_unique_jobs/lock.rb +325 -0
  28. data/lib/sidekiq_unique_jobs/lock_args.rb +123 -0
  29. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  30. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  31. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  32. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  33. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  34. data/lib/sidekiq_unique_jobs/locksmith.rb +275 -102
  35. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  36. data/lib/sidekiq_unique_jobs/logging.rb +188 -33
  37. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  38. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  39. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  40. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  41. data/lib/sidekiq_unique_jobs/lua/lock.lua +93 -0
  42. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  43. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  44. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +94 -0
  45. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  46. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  54. data/lib/sidekiq_unique_jobs/lua/unlock.lua +102 -0
  55. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  56. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  57. data/lib/sidekiq_unique_jobs/middleware/client.rb +40 -0
  58. data/lib/sidekiq_unique_jobs/middleware/server.rb +29 -0
  59. data/lib/sidekiq_unique_jobs/middleware.rb +29 -31
  60. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  61. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  62. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  63. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  64. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +61 -15
  65. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  66. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  67. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  68. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  69. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +35 -32
  70. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  71. data/lib/sidekiq_unique_jobs/orphans/manager.rb +248 -0
  72. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  73. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  74. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  75. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  76. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +231 -0
  77. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  78. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  79. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  80. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  81. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  82. data/lib/sidekiq_unique_jobs/redis/string.rb +49 -0
  83. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  84. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  85. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  86. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  87. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  88. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  89. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  90. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  91. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +114 -65
  92. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +241 -35
  93. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +18 -16
  94. data/lib/sidekiq_unique_jobs/testing.rb +62 -21
  95. data/lib/sidekiq_unique_jobs/timer_task.rb +78 -0
  96. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  97. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  98. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  99. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  100. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  101. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  102. data/lib/sidekiq_unique_jobs/web/helpers.rb +128 -13
  103. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  104. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  105. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  106. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  107. data/lib/sidekiq_unique_jobs/web.rb +57 -27
  108. data/lib/sidekiq_unique_jobs.rb +52 -7
  109. data/lib/tasks/changelog.rake +15 -15
  110. metadata +124 -184
  111. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  112. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  113. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  114. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  115. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  116. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  117. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  118. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  119. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  120. data/redis/acquire_lock.lua +0 -21
  121. data/redis/convert_legacy_lock.lua +0 -13
  122. data/redis/delete.lua +0 -14
  123. data/redis/delete_by_digest.lua +0 -23
  124. data/redis/delete_job_by_digest.lua +0 -60
  125. data/redis/lock.lua +0 -62
  126. data/redis/release_stale_locks.lua +0 -90
  127. data/redis/unlock.lua +0 -35
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Handles loading and dumping of json
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ module JSON
8
+ module_function
9
+
10
+ #
11
+ # Parses a JSON string into an object
12
+ #
13
+ # @param [String] string the object to parse
14
+ #
15
+ # @return [Object]
16
+ #
17
+ def load_json(string)
18
+ return if string.nil? || string.empty?
19
+
20
+ ::JSON.parse(string)
21
+ end
22
+
23
+ #
24
+ # Prevents trying JSON.load from raising errors given argument is a hash
25
+ #
26
+ # @param [String, Hash] string the JSON string to parse
27
+ #
28
+ # @return [Hash,Array]
29
+ #
30
+ def safe_load_json(string)
31
+ return string if string.is_a?(Hash)
32
+
33
+ load_json(string)
34
+ end
35
+
36
+ #
37
+ # Dumps an object into a JSON string
38
+ #
39
+ # @param [Object] object a JSON convertible object
40
+ #
41
+ # @return [String] a JSON string
42
+ #
43
+ def dump_json(object)
44
+ ::JSON.generate(object)
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Key class wraps logic dealing with various lock keys
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class Key
8
+ #
9
+ # @!attribute [r] digest
10
+ # @return [String] the digest key for which keys are created
11
+ attr_reader :digest
12
+ #
13
+ # @!attribute [r] queued
14
+ # @return [String] the list key with queued job_id's
15
+ attr_reader :queued
16
+ #
17
+ # @!attribute [r] primed
18
+ # @return [String] the list key with primed job_id's
19
+ attr_reader :primed
20
+ #
21
+ # @!attribute [r] locked
22
+ # @return [String] the hash key with locked job_id's
23
+ attr_reader :locked
24
+ #
25
+ # @!attribute [r] info
26
+ # @return [String] information about the lock
27
+ attr_reader :info
28
+ #
29
+ # @!attribute [r] changelog
30
+ # @return [String] the zset with changelog entries
31
+ attr_reader :changelog
32
+ #
33
+ # @!attribute [r] digests
34
+ # @return [String] the zset with locked digests
35
+ attr_reader :digests
36
+
37
+ #
38
+ # Initialize a new Key
39
+ #
40
+ # @param [String] digest the digest to use as key
41
+ #
42
+ def initialize(digest)
43
+ @digest = digest
44
+ @queued = suffixed_key("QUEUED")
45
+ @primed = suffixed_key("PRIMED")
46
+ @locked = suffixed_key("LOCKED")
47
+ @info = suffixed_key("INFO")
48
+ @changelog = CHANGELOGS
49
+ @digests = DIGESTS
50
+ end
51
+
52
+ #
53
+ # Provides the only important information about this keys
54
+ #
55
+ #
56
+ # @return [String]
57
+ #
58
+ def to_s
59
+ digest
60
+ end
61
+
62
+ # @see to_s
63
+ def inspect
64
+ digest
65
+ end
66
+
67
+ #
68
+ # Compares keys by digest
69
+ #
70
+ # @param [Key] other the key to compare with
71
+ #
72
+ # @return [true, false]
73
+ #
74
+ def ==(other)
75
+ digest == other.digest
76
+ end
77
+
78
+ #
79
+ # Returns all keys as an ordered array
80
+ #
81
+ # @return [Array] an ordered array with all keys
82
+ #
83
+ def to_a
84
+ [digest, queued, primed, locked, info, changelog, digests]
85
+ end
86
+
87
+ private
88
+
89
+ def suffixed_key(variable)
90
+ "#{digest}:#{variable}"
91
+ end
92
+ end
93
+ end
@@ -5,10 +5,33 @@ module SidekiqUniqueJobs
5
5
  # Abstract base class for locks
6
6
  #
7
7
  # @abstract
8
- # @author Mikael Henriksson <mikael@zoolutions.se>
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
9
  class BaseLock
10
+ extend Forwardable
11
+
12
+ # includes "SidekiqUniqueJobs::Logging"
13
+ # @!parse include SidekiqUniqueJobs::Logging
10
14
  include SidekiqUniqueJobs::Logging
11
15
 
16
+ # includes "SidekiqUniqueJobs::Reflectable"
17
+ # @!parse include SidekiqUniqueJobs::Reflectable
18
+ include SidekiqUniqueJobs::Reflectable
19
+
20
+ #
21
+ # Validates that the sidekiq_options for the worker is valid
22
+ #
23
+ # @param [Hash] options the sidekiq_options given to the worker
24
+ #
25
+ # @return [void]
26
+ #
27
+ def self.validate_options(options = {})
28
+ Validator.validate(options)
29
+ end
30
+
31
+ # NOTE: Mainly used for a clean testing API
32
+ #
33
+ def_delegators :locksmith, :locked?
34
+
12
35
  # @param [Hash] item the Sidekiq job hash
13
36
  # @param [Proc] callback the callback to use after unlock
14
37
  # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -16,21 +39,22 @@ module SidekiqUniqueJobs
16
39
  @item = item
17
40
  @callback = callback
18
41
  @redis_pool = redis_pool
19
- add_uniqueness_when_missing # Used to ease testing
42
+ @attempt = 0
43
+ prepare_item # Used to ease testing
44
+ @lock_config = LockConfig.new(item)
20
45
  end
21
46
 
22
- # Handles locking of sidekiq jobs.
23
- # Will call a conflict strategy if lock can't be achieved.
24
- # @return [String] the sidekiq job id
47
+ #
48
+ # Locks a sidekiq job
49
+ #
50
+ # @note Will call a conflict strategy if lock can't be achieved.
51
+ #
52
+ # @return [String, nil] the locked jid when properly locked, else nil.
53
+ #
54
+ # @yield to the caller when given a block
55
+ #
25
56
  def lock
26
- @attempt = 0
27
- return item[JID_KEY] if locked?
28
-
29
- if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
30
- token
31
- else
32
- call_strategy
33
- end
57
+ raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
34
58
  end
35
59
 
36
60
  # Execute the job in the Sidekiq server processor
@@ -39,97 +63,102 @@ module SidekiqUniqueJobs
39
63
  raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
40
64
  end
41
65
 
42
- # Unlocks the job from redis
43
- # @return [String] sidekiq job id when successful
44
- # @return [false] when unsuccessful
45
- def unlock
46
- locksmith.unlock(item[JID_KEY]) # Only signal to release the lock
47
- end
48
-
49
- # Deletes the job from redis if it is locked.
50
- def delete
51
- locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
52
- end
53
-
54
- # Forcefully deletes the job from redis.
55
- # This is good for jobs when a previous lock was not unlocked
56
- def delete!
57
- locksmith.delete! # Force delete the lock
58
- end
59
-
60
- # Checks if the item has achieved a lock
61
- # @return [true] when this jid has locked the job
62
- # @return [false] when this jid has not locked the job
63
- def locked?
64
- locksmith.locked?(item[JID_KEY])
66
+ #
67
+ # The lock manager/client
68
+ #
69
+ # @api private
70
+ # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
71
+ #
72
+ def locksmith
73
+ @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
65
74
  end
66
75
 
67
76
  private
68
77
 
69
- def add_uniqueness_when_missing
70
- return if item.key?(UNIQUE_DIGEST_KEY)
71
-
72
- # The below should only be done to ease testing
73
- # in production this will be done by the middleware
74
- SidekiqUniqueJobs::Job.add_uniqueness(item)
75
- end
76
-
77
- def call_strategy
78
- @attempt += 1
79
- strategy.call { lock if replace? }
80
- end
81
-
82
- def replace?
83
- strategy.replace? && attempt < 2
84
- end
85
-
86
- # The sidekiq job hash
87
- # @return [Hash] the Sidekiq job hash
78
+ # @!attribute [r] item
79
+ # @return [Hash<String, Object>] the Sidekiq job hash
88
80
  attr_reader :item
89
-
90
- # The sidekiq redis pool
91
- # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
81
+ # @!attribute [r] lock_config
82
+ # @return [LockConfig] a lock configuration
83
+ attr_reader :lock_config
84
+ # @!attribute [r] redis_pool
85
+ # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
92
86
  attr_reader :redis_pool
93
-
94
- # The sidekiq job hash
95
- # @return [Proc] the callback to use after unlock
87
+ # @!attribute [r] callback
88
+ # @return [Proc] the block to call after unlock
96
89
  attr_reader :callback
97
-
98
- # The current attempt to lock the job
99
- # @return [Integer] the numerical value of the attempt
90
+ # @!attribute [r] attempt
91
+ # @return [Integer] the current locking attempt
100
92
  attr_reader :attempt
101
93
 
102
- # The interface to the locking mechanism
103
- # @return [SidekiqUniqueJobs::Locksmith]
104
- def locksmith
105
- @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
94
+ #
95
+ # Eases testing by allowing the lock implementation to add the missing
96
+ # keys to the job hash.
97
+ #
98
+ #
99
+ # @return [void] the return value should be irrelevant
100
+ #
101
+ def prepare_item
102
+ return if item.key?(LOCK_DIGEST)
103
+
104
+ # The below should only be done to ease testing
105
+ # in production this will be done by the middleware
106
+ SidekiqUniqueJobs::Job.prepare(item)
106
107
  end
107
108
 
108
- def with_cleanup
109
- yield
110
- rescue Sidekiq::Shutdown
111
- log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
112
- raise
113
- else
114
- unlock_with_callback
109
+ #
110
+ # Call whatever strategry that has been configured
111
+ #
112
+ # @param [Symbol] origin: the origin `:client` or `:server`
113
+ #
114
+ # @return [void] the return value is irrelevant
115
+ #
116
+ # @yieldparam [void] if a new job id was set and a block is given
117
+ # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
118
+ # one specific situation to yield back to the middleware.
119
+ def call_strategy(origin:)
120
+ new_job_id = nil
121
+ strategy = strategy_for(origin)
122
+ @attempt += 1
123
+
124
+ strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
125
+ yield if new_job_id && block_given?
115
126
  end
116
127
 
117
- def unlock_with_callback
118
- return log_warn("might need to be unlocked manually") unless unlock
128
+ def unlock_and_callback
129
+ return callback_safely if locksmith.unlock
119
130
 
120
- callback_safely
131
+ reflect(:unlock_failed, item)
121
132
  end
122
133
 
123
134
  def callback_safely
124
135
  callback&.call
125
- item[JID_KEY]
136
+ item[JID]
126
137
  rescue StandardError
127
- log_warn("unlocked successfully but the #after_unlock callback failed!")
138
+ reflect(:after_unlock_callback_failed, item)
128
139
  raise
129
140
  end
130
141
 
131
- def strategy
132
- @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT_KEY]).new(item)
142
+ def strategy_for(origin)
143
+ case origin
144
+ when :client
145
+ client_strategy
146
+ when :server
147
+ server_strategy
148
+ else
149
+ raise SidekiqUniqueJobs::InvalidArgument,
150
+ "#origin needs to be either `:server` or `:client`"
151
+ end
152
+ end
153
+
154
+ def client_strategy
155
+ @client_strategy ||=
156
+ OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
157
+ end
158
+
159
+ def server_strategy
160
+ @server_strategy ||=
161
+ OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
133
162
  end
134
163
  end
135
164
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq client process
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class ClientValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq client process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_client_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
24
+ lock_config
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq server process
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class ServerValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:replace].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq server process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_server_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
24
+ end
25
+ end
26
+ end
27
+ end
@@ -11,36 +11,61 @@ module SidekiqUniqueJobs
11
11
  # See {#lock} for more information about the client.
12
12
  # See {#execute} for more information about the server
13
13
  #
14
- # @author Mikael Henriksson <mikael@zoolutions.se>
14
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
15
15
  class UntilAndWhileExecuting < BaseLock
16
+ #
17
+ # Locks a sidekiq job
18
+ #
19
+ # @note Will call a conflict strategy if lock can't be achieved.
20
+ #
21
+ # @return [String, nil] the locked jid when properly locked, else nil.
22
+ #
23
+ # @yield to the caller when given a block
24
+ #
25
+ def lock(origin: :client, &block)
26
+ unless (token = locksmith.lock)
27
+ reflect(:lock_failed, item)
28
+ call_strategy(origin: origin, &block)
29
+
30
+ return
31
+ end
32
+
33
+ yield if block
34
+
35
+ token
36
+ end
37
+
16
38
  # Executes in the Sidekiq server process
17
39
  # @yield to the worker class perform method
18
40
  def execute
19
- if unlock
20
- lock_on_failure do
21
- runtime_lock.execute { return yield }
22
- end
41
+ if locksmith.unlock
42
+ # ensure_relocked do
43
+ runtime_lock.execute { return yield }
44
+ # end
23
45
  else
24
- log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST_KEY]} #{item[JID_KEY]}"
46
+ reflect(:unlock_failed, item)
25
47
  end
26
- ensure
27
- runtime_lock.delete!
28
- end
48
+ rescue Exception # rubocop:disable Lint/RescueException
49
+ reflect(:execution_failed, item)
50
+ locksmith.lock(wait: 2)
29
51
 
30
- def runtime_lock
31
- @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
52
+ raise
32
53
  end
33
54
 
34
55
  private
35
56
 
36
- def lock_on_failure
57
+ def ensure_relocked
37
58
  yield
38
- runtime_lock.delete!
39
59
  rescue Exception # rubocop:disable Lint/RescueException
40
- log_error("Failed to execute job, restoring lock")
41
- lock
60
+ reflect(:execution_failed, item)
61
+ locksmith.lock
62
+
42
63
  raise
43
64
  end
65
+
66
+ def runtime_lock
67
+ @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
68
+ end
44
69
  end
45
70
  end
46
71
  end
@@ -6,18 +6,36 @@ module SidekiqUniqueJobs
6
6
  # - Locks on perform_in or perform_async
7
7
  # - Unlocks after yielding to the worker's perform method
8
8
  #
9
- # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
10
  class UntilExecuted < BaseLock
11
- OK ||= "OK"
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ # @yield to the caller when given a block
19
+ #
20
+ def lock(&block)
21
+ unless (token = locksmith.lock)
22
+ reflect(:lock_failed, item)
23
+ call_strategy(origin: :client, &block)
24
+
25
+ return
26
+ end
27
+
28
+ yield if block
29
+
30
+ token
31
+ end
12
32
 
13
33
  # Executes in the Sidekiq server process
14
34
  # @yield to the worker class perform method
15
35
  def execute
16
- if locked?
17
- with_cleanup { yield }
18
- else
19
- log_warn "the unique_key: #{item[UNIQUE_DIGEST_KEY]} is not locked, allowing job to silently complete"
20
- nil
36
+ locksmith.execute do
37
+ yield
38
+ unlock_and_callback
21
39
  end
22
40
  end
23
41
  end
@@ -6,12 +6,32 @@ module SidekiqUniqueJobs
6
6
  # - Locks on perform_in or perform_async
7
7
  # - Unlocks before yielding to the worker's perform method
8
8
  #
9
- # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
10
  class UntilExecuting < BaseLock
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ def lock(&block)
19
+ unless (token = locksmith.lock)
20
+ reflect(:lock_failed, item)
21
+ call_strategy(origin: :client, &block)
22
+
23
+ return
24
+ end
25
+
26
+ yield if block
27
+
28
+ token
29
+ end
30
+
11
31
  # Executes in the Sidekiq server process
12
32
  # @yield to the worker class perform method
13
33
  def execute
14
- unlock_with_callback
34
+ callback_safely if locksmith.unlock
15
35
  yield
16
36
  end
17
37
  end
@@ -2,28 +2,38 @@
2
2
 
3
3
  module SidekiqUniqueJobs
4
4
  class Lock
5
- # Locks jobs until the lock has expired
6
- # - Locks on perform_in or perform_async
7
- # - Unlocks when the expiration is hit
8
5
  #
9
- # See {#lock} for more information about the client.
10
- # See {#execute} for more information about the server
6
+ # UntilExpired locks until the job expires
11
7
  #
12
- # @author Mikael Henriksson <mikael@zoolutions.se>
13
- class UntilExpired < BaseLock
14
- # Prevents these locks from being unlocked
15
- # @return [true] always returns true
16
- def unlock
17
- true
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class UntilExpired < UntilExecuted
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ # @yield to the caller when given a block
19
+ #
20
+ def lock(&block)
21
+ unless (token = locksmith.lock)
22
+ reflect(:lock_failed, item)
23
+ call_strategy(origin: :client, &block)
24
+
25
+ return
26
+ end
27
+
28
+ yield if block
29
+
30
+ token
18
31
  end
19
32
 
20
33
  # Executes in the Sidekiq server process
21
34
  # @yield to the worker class perform method
22
- def execute
23
- return unless locked?
24
-
25
- yield
26
- # this lock does not handle after_unlock since we don't know when that would happen
35
+ def execute(&block)
36
+ locksmith.execute(&block)
27
37
  end
28
38
  end
29
39
  end