sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Key class wraps logic dealing with various lock keys
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class Key
8
+ #
9
+ # @!attribute [r] digest
10
+ # @return [String] the digest key for which keys are created
11
+ attr_reader :digest
12
+ #
13
+ # @!attribute [r] queued
14
+ # @return [String] the list key with queued job_id's
15
+ attr_reader :queued
16
+ #
17
+ # @!attribute [r] primed
18
+ # @return [String] the list key with primed job_id's
19
+ attr_reader :primed
20
+ #
21
+ # @!attribute [r] locked
22
+ # @return [String] the hash key with locked job_id's
23
+ attr_reader :locked
24
+ #
25
+ # @!attribute [r] info
26
+ # @return [String] information about the lock
27
+ attr_reader :info
28
+ #
29
+ # @!attribute [r] changelog
30
+ # @return [String] the zset with changelog entries
31
+ attr_reader :changelog
32
+ #
33
+ # @!attribute [r] digests
34
+ # @return [String] the zset with locked digests
35
+ attr_reader :digests
36
+ #
37
+ # @!attribute [r] expiring_digests
38
+ # @return [String] the zset with locked expiring_digests
39
+ attr_reader :expiring_digests
40
+
41
+ #
42
+ # Initialize a new Key
43
+ #
44
+ # @param [String] digest the digest to use as key
45
+ #
46
+ def initialize(digest)
47
+ @digest = digest
48
+ @queued = suffixed_key("QUEUED")
49
+ @primed = suffixed_key("PRIMED")
50
+ @locked = suffixed_key("LOCKED")
51
+ @info = suffixed_key("INFO")
52
+ @changelog = CHANGELOGS
53
+ @digests = DIGESTS
54
+ @expiring_digests = EXPIRING_DIGESTS
55
+ end
56
+
57
+ #
58
+ # Provides the only important information about this keys
59
+ #
60
+ #
61
+ # @return [String]
62
+ #
63
+ def to_s
64
+ digest
65
+ end
66
+
67
+ # @see to_s
68
+ def inspect
69
+ digest
70
+ end
71
+
72
+ #
73
+ # Compares keys by digest
74
+ #
75
+ # @param [Key] other the key to compare with
76
+ #
77
+ # @return [true, false]
78
+ #
79
+ def ==(other)
80
+ digest == other.digest
81
+ end
82
+
83
+ #
84
+ # Returns all keys as an ordered array
85
+ #
86
+ # @return [Array] an ordered array with all keys
87
+ #
88
+ def to_a
89
+ [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
90
+ end
91
+
92
+ private
93
+
94
+ def suffixed_key(variable)
95
+ "#{digest}:#{variable}"
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Abstract base class for locks
6
+ #
7
+ # @abstract
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ class BaseLock
10
+ extend Forwardable
11
+
12
+ # includes "SidekiqUniqueJobs::Logging"
13
+ # @!parse include SidekiqUniqueJobs::Logging
14
+ include SidekiqUniqueJobs::Logging
15
+
16
+ # includes "SidekiqUniqueJobs::Reflectable"
17
+ # @!parse include SidekiqUniqueJobs::Reflectable
18
+ include SidekiqUniqueJobs::Reflectable
19
+
20
+ #
21
+ # Validates that the sidekiq_options for the worker is valid
22
+ #
23
+ # @param [Hash] options the sidekiq_options given to the worker
24
+ #
25
+ # @return [void]
26
+ #
27
+ def self.validate_options(options = {})
28
+ Validator.validate(options)
29
+ end
30
+
31
+ # NOTE: Mainly used for a clean testing API
32
+ #
33
+ def_delegators :locksmith, :locked?
34
+
35
+ # @param [Hash] item the Sidekiq job hash
36
+ # @param [Proc] callback the callback to use after unlock
37
+ # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
38
+ def initialize(item, callback, redis_pool = nil)
39
+ @item = item
40
+ @callback = callback
41
+ @redis_pool = redis_pool
42
+ @attempt = 0
43
+ prepare_item # Used to ease testing
44
+ @lock_config = LockConfig.new(item)
45
+ end
46
+
47
+ #
48
+ # Locks a sidekiq job
49
+ #
50
+ # @note Will call a conflict strategy if lock can't be achieved.
51
+ #
52
+ # @return [String, nil] the locked jid when properly locked, else nil.
53
+ #
54
+ # @yield to the caller when given a block
55
+ #
56
+ def lock
57
+ raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
58
+ end
59
+
60
+ # Execute the job in the Sidekiq server processor
61
+ # @raise [NotImplementedError] needs to be implemented in child class
62
+ def execute
63
+ raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
64
+ end
65
+
66
+ #
67
+ # The lock manager/client
68
+ #
69
+ # @api private
70
+ # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
71
+ #
72
+ def locksmith
73
+ @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
74
+ end
75
+
76
+ private
77
+
78
+ # @!attribute [r] item
79
+ # @return [Hash<String, Object>] the Sidekiq job hash
80
+ attr_reader :item
81
+ # @!attribute [r] lock_config
82
+ # @return [LockConfig] a lock configuration
83
+ attr_reader :lock_config
84
+ # @!attribute [r] redis_pool
85
+ # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
86
+ attr_reader :redis_pool
87
+ # @!attribute [r] callback
88
+ # @return [Proc] the block to call after unlock
89
+ attr_reader :callback
90
+ # @!attribute [r] attempt
91
+ # @return [Integer] the current locking attempt
92
+ attr_reader :attempt
93
+
94
+ #
95
+ # Eases testing by allowing the lock implementation to add the missing
96
+ # keys to the job hash.
97
+ #
98
+ #
99
+ # @return [void] the return value should be irrelevant
100
+ #
101
+ def prepare_item
102
+ return if item.key?(LOCK_DIGEST)
103
+
104
+ # The below should only be done to ease testing
105
+ # in production this will be done by the middleware
106
+ SidekiqUniqueJobs::Job.prepare(item)
107
+ end
108
+
109
+ #
110
+ # Call whatever strategry that has been configured
111
+ #
112
+ # @param [Symbol] origin the origin `:client` or `:server`
113
+ #
114
+ # @return [void] the return value is irrelevant
115
+ #
116
+ # @yieldparam [void] if a new job id was set and a block is given
117
+ # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
118
+ # one specific situation to yield back to the middleware.
119
+ def call_strategy(origin:)
120
+ new_job_id = nil
121
+ strategy = strategy_for(origin)
122
+ @attempt += 1
123
+
124
+ strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
125
+ yield if new_job_id && block_given?
126
+ end
127
+
128
+ def unlock_and_callback
129
+ return callback_safely if locksmith.unlock
130
+
131
+ reflect(:unlock_failed, item)
132
+ end
133
+
134
+ def callback_safely
135
+ callback&.call
136
+ item[JID]
137
+ rescue StandardError
138
+ reflect(:after_unlock_callback_failed, item)
139
+ raise
140
+ end
141
+
142
+ def strategy_for(origin)
143
+ case origin
144
+ when :client
145
+ client_strategy
146
+ when :server
147
+ server_strategy
148
+ else
149
+ raise SidekiqUniqueJobs::InvalidArgument,
150
+ "#origin needs to be either `:server` or `:client`"
151
+ end
152
+ end
153
+
154
+ def client_strategy
155
+ @client_strategy ||=
156
+ OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
157
+ end
158
+
159
+ def server_strategy
160
+ @server_strategy ||=
161
+ OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq client process
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class ClientValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq client process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_client_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
24
+ lock_config
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validates the sidekiq options for the Sidekiq server process
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class ServerValidator
11
+ #
12
+ # @return [Array<Symbol>] a collection of invalid conflict resolutions
13
+ INVALID_ON_CONFLICTS = [:replace].freeze
14
+
15
+ #
16
+ # Validates the sidekiq options for the Sidekiq server process
17
+ #
18
+ #
19
+ def self.validate(lock_config)
20
+ on_conflict = lock_config.on_server_conflict
21
+ return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
22
+
23
+ lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Locks jobs while the job is executing in the server process
6
+ # - Locks on perform_in or perform_async (see {UntilExecuting})
7
+ # - Unlocks before yielding to the worker's perform method (see {UntilExecuting})
8
+ # - Locks before yielding to the worker's perform method (see {WhileExecuting})
9
+ # - Unlocks after yielding to the worker's perform method (see {WhileExecuting})
10
+ #
11
+ # See {#lock} for more information about the client.
12
+ # See {#execute} for more information about the server
13
+ #
14
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
15
+ class UntilAndWhileExecuting < BaseLock
16
+ #
17
+ # Locks a sidekiq job
18
+ #
19
+ # @note Will call a conflict strategy if lock can't be achieved.
20
+ #
21
+ # @return [String, nil] the locked jid when properly locked, else nil.
22
+ #
23
+ # @yield to the caller when given a block
24
+ #
25
+ def lock(origin: :client, &block)
26
+ unless (token = locksmith.lock)
27
+ reflect(:lock_failed, item)
28
+ call_strategy(origin: origin, &block)
29
+
30
+ return
31
+ end
32
+
33
+ yield if block
34
+
35
+ token
36
+ end
37
+
38
+ # Executes in the Sidekiq server process
39
+ # @yield to the worker class perform method
40
+ def execute
41
+ if locksmith.unlock
42
+ # ensure_relocked do
43
+ runtime_lock.execute { return yield }
44
+ # end
45
+ else
46
+ reflect(:unlock_failed, item)
47
+ end
48
+ rescue Exception # rubocop:disable Lint/RescueException
49
+ reflect(:execution_failed, item)
50
+ locksmith.lock(wait: 2)
51
+
52
+ raise
53
+ end
54
+
55
+ private
56
+
57
+ def ensure_relocked
58
+ yield
59
+ rescue Exception # rubocop:disable Lint/RescueException
60
+ reflect(:execution_failed, item)
61
+ locksmith.lock
62
+
63
+ raise
64
+ end
65
+
66
+ def runtime_lock
67
+ @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Locks jobs until the server is done executing the job
6
+ # - Locks on perform_in or perform_async
7
+ # - Unlocks after yielding to the worker's perform method
8
+ #
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
+ class UntilExecuted < BaseLock
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ # @yield to the caller when given a block
19
+ #
20
+ def lock(&block)
21
+ unless (token = locksmith.lock)
22
+ reflect(:lock_failed, item)
23
+ call_strategy(origin: :client, &block)
24
+
25
+ return
26
+ end
27
+
28
+ yield if block
29
+
30
+ token
31
+ end
32
+
33
+ # Executes in the Sidekiq server process
34
+ # @yield to the worker class perform method
35
+ def execute
36
+ executed = locksmith.execute do
37
+ yield
38
+ ensure
39
+ unlock_and_callback
40
+ end
41
+
42
+ reflect(:execution_failed, item) unless executed
43
+
44
+ nil
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Locks jobs until {#execute} starts
6
+ # - Locks on perform_in or perform_async
7
+ # - Unlocks before yielding to the worker's perform method
8
+ #
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
+ class UntilExecuting < BaseLock
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ def lock(&block)
19
+ unless (token = locksmith.lock)
20
+ reflect(:lock_failed, item)
21
+ call_strategy(origin: :client, &block)
22
+
23
+ return
24
+ end
25
+
26
+ yield if block
27
+
28
+ token
29
+ end
30
+
31
+ # Executes in the Sidekiq server process
32
+ # @yield to the worker class perform method
33
+ def execute
34
+ callback_safely if locksmith.unlock
35
+ yield
36
+ rescue StandardError => ex
37
+ reflect(:execution_failed, item, ex)
38
+ locksmith.lock(wait: 1)
39
+ raise
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # UntilExpired locks until the job expires
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class UntilExpired < UntilExecuted
11
+ #
12
+ # Locks a sidekiq job
13
+ #
14
+ # @note Will call a conflict strategy if lock can't be achieved.
15
+ #
16
+ # @return [String, nil] the locked jid when properly locked, else nil.
17
+ #
18
+ # @yield to the caller when given a block
19
+ #
20
+ def lock(&block)
21
+ unless (token = locksmith.lock)
22
+ reflect(:lock_failed, item)
23
+ call_strategy(origin: :client, &block)
24
+
25
+ return
26
+ end
27
+
28
+ yield if block
29
+
30
+ token
31
+ end
32
+
33
+ # Executes in the Sidekiq server process
34
+ # @yield to the worker class perform method
35
+ def execute(&block)
36
+ executed = locksmith.execute(&block)
37
+
38
+ reflect(:execution_failed, item) unless executed
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ #
6
+ # Validator base class to avoid some duplication
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Validator
11
+ #
12
+ # @return [Hash] a hash mapping of deprecated keys and their new value
13
+ DEPRECATED_KEYS = {
14
+ UNIQUE.to_sym => LOCK.to_sym,
15
+ UNIQUE_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
16
+ LOCK_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
17
+ UNIQUE_PREFIX.to_sym => LOCK_PREFIX.to_sym,
18
+ }.freeze
19
+
20
+ #
21
+ # Shorthand for `new(options).validate`
22
+ #
23
+ # @param [Hash] options the sidekiq_options for the worker being validated
24
+ #
25
+ # @return [LockConfig] the lock configuration with errors if any
26
+ #
27
+ def self.validate(options)
28
+ new(options).validate
29
+ end
30
+
31
+ #
32
+ # @!attribute [r] lock_config
33
+ # @return [LockConfig] the lock configuration for this worker
34
+ attr_reader :lock_config
35
+
36
+ #
37
+ # Initialize a new validator
38
+ #
39
+ # @param [Hash] options the sidekiq_options for the worker being validated
40
+ #
41
+ def initialize(options)
42
+ @options = options.transform_keys(&:to_sym)
43
+ @lock_config = LockConfig.new(options)
44
+ handle_deprecations
45
+ end
46
+
47
+ #
48
+ # Validate the workers lock configuration
49
+ #
50
+ #
51
+ # @return [LockConfig] the lock configuration with errors if any
52
+ #
53
+ def validate
54
+ case lock_config.type
55
+ when :while_executing
56
+ validate_server
57
+ when :until_executing
58
+ validate_client
59
+ else
60
+ validate_client
61
+ validate_server
62
+ end
63
+
64
+ lock_config
65
+ end
66
+
67
+ #
68
+ # Validate deprecated keys
69
+ # adds useful information about how to proceed with fixing handle_deprecations
70
+ #
71
+ # @return [void]
72
+ #
73
+ def handle_deprecations
74
+ DEPRECATED_KEYS.each do |old, new|
75
+ next unless @options.key?(old)
76
+
77
+ lock_config.errors[old] = "is deprecated, use `#{new}: #{@options[old]}` instead."
78
+ end
79
+ end
80
+
81
+ #
82
+ # Validates the client configuration
83
+ #
84
+ def validate_client
85
+ ClientValidator.validate(lock_config)
86
+ end
87
+
88
+ #
89
+ # Validates the server configuration
90
+ #
91
+ def validate_server
92
+ ServerValidator.validate(lock_config)
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Locks jobs while the job is executing in the server process
6
+ # - Locks before yielding to the worker's perform method
7
+ # - Unlocks after yielding to the worker's perform method
8
+ #
9
+ # See {#lock} for more information about the client.
10
+ # See {#execute} for more information about the server
11
+ #
12
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
13
+ class WhileExecuting < BaseLock
14
+ #
15
+ # @return [String] returns :RUN
16
+ RUN_SUFFIX = ":RUN"
17
+
18
+ include SidekiqUniqueJobs::OptionsWithFallback
19
+ include SidekiqUniqueJobs::Logging::Middleware
20
+
21
+ # @param [Hash] item the Sidekiq job hash
22
+ # @param [Proc] callback callback to call after unlock
23
+ # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
24
+ #
25
+ def initialize(item, callback, redis_pool = nil)
26
+ super(item, callback, redis_pool)
27
+ append_unique_key_suffix
28
+ end
29
+
30
+ # Simulate that a client lock was achieved.
31
+ # These locks should only ever be created in the server process.
32
+ # @return [true] always returns true
33
+ def lock
34
+ job_id = item[JID]
35
+ yield if block_given?
36
+
37
+ job_id
38
+ end
39
+
40
+ # Executes in the Sidekiq server process.
41
+ # These jobs are locked in the server process not from the client
42
+ # @yield to the worker class perform method
43
+ def execute(&block)
44
+ with_logging_context do
45
+ executed = locksmith.execute do
46
+ yield
47
+ item[JID]
48
+ ensure
49
+ unlock_and_callback
50
+ end
51
+
52
+ unless executed
53
+ reflect(:execution_failed, item)
54
+ call_strategy(origin: :server, &block)
55
+ end
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ # This is safe as the base_lock always creates a new digest
62
+ # The append there for needs to be done every time
63
+ def append_unique_key_suffix
64
+ return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
65
+
66
+ item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
67
+ end
68
+ end
69
+ end
70
+ end