sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ class Lock
5
+ # Locks jobs while executing
6
+ # Locks from the server process
7
+ # Unlocks after the server is done processing
8
+ #
9
+ # See {#lock} for more information about the client.
10
+ # See {#execute} for more information about the server
11
+ #
12
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
13
+ class WhileExecutingReject < WhileExecuting
14
+ # Overridden with a forced {OnConflict::Reject} strategy
15
+ # @return [OnConflict::Reject] a reject strategy
16
+ def server_strategy
17
+ @server_strategy ||= OnConflict.find_strategy(:reject).new(item, redis_pool)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,348 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class Lock provides access to information about a lock
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class Lock # rubocop:disable Metrics/ClassLength
10
+ # includes "SidekiqUniqueJobs::Connection"
11
+ # @!parse include SidekiqUniqueJobs::Connection
12
+ include SidekiqUniqueJobs::Connection
13
+
14
+ # includes "SidekiqUniqueJobs::Timing"
15
+ # @!parse include SidekiqUniqueJobs::Timing
16
+ include SidekiqUniqueJobs::Timing
17
+
18
+ # includes "SidekiqUniqueJobs::JSON"
19
+ # @!parse include SidekiqUniqueJobs::JSON
20
+ include SidekiqUniqueJobs::JSON
21
+
22
+ #
23
+ # @!attribute [r] key
24
+ # @return [String] the entity redis key
25
+ attr_reader :key
26
+
27
+ #
28
+ # Initialize a locked lock
29
+ #
30
+ # @param [String] digest a unique digest
31
+ # @param [String] job_id a sidekiq JID
32
+ # @param [Hash] lock_info information about the lock
33
+ #
34
+ # @return [Lock] a newly lock that has been locked
35
+ #
36
+ def self.create(digest, job_id, lock_info: {}, time: Timing.now_f, score: nil)
37
+ lock = new(digest, time: time)
38
+ lock.lock(job_id, lock_info, score)
39
+ lock
40
+ end
41
+
42
+ #
43
+ # Initialize a new lock
44
+ #
45
+ # @param [String, Key] key either a digest or an instance of a {Key}
46
+ # @param [Timstamp, Float] time nil optional timestamp to initiate this lock with
47
+ #
48
+ def initialize(key, time: nil)
49
+ @key = get_key(key)
50
+ time = time.to_f unless time.is_a?(Float)
51
+ return unless time.nonzero?
52
+
53
+ @created_at = time
54
+ end
55
+
56
+ #
57
+ # Locks a job_id
58
+ #
59
+ # @note intended only for testing purposes
60
+ #
61
+ # @param [String] job_id a sidekiq JID
62
+ # @param [Hash] lock_info information about the lock
63
+ #
64
+ # @return [void]
65
+ #
66
+ def lock(job_id, lock_info = {}, score = nil)
67
+ score ||= now_f
68
+ redis do |conn|
69
+ conn.multi do |pipeline|
70
+ pipeline.set(key.digest, job_id)
71
+ pipeline.hset(key.locked, job_id, now_f)
72
+ info.set(lock_info, pipeline)
73
+ add_digest_to_set(pipeline, lock_info, score)
74
+ pipeline.zadd(key.changelog, score, changelog_json(job_id, "queue.lua", "Queued"))
75
+ pipeline.zadd(key.changelog, score, changelog_json(job_id, "lock.lua", "Locked"))
76
+ end
77
+ end
78
+ end
79
+
80
+ #
81
+ # Create the :QUEUED key
82
+ #
83
+ # @note intended only for testing purposes
84
+ #
85
+ # @param [String] job_id a sidekiq JID
86
+ #
87
+ # @return [void]
88
+ #
89
+ def queue(job_id)
90
+ redis do |conn|
91
+ conn.lpush(key.queued, job_id)
92
+ end
93
+ end
94
+
95
+ #
96
+ # Create the :PRIMED key
97
+ #
98
+ # @note intended only for testing purposes
99
+ #
100
+ # @param [String] job_id a sidekiq JID
101
+ #
102
+ # @return [void]
103
+ #
104
+ def prime(job_id)
105
+ redis do |conn|
106
+ conn.lpush(key.primed, job_id)
107
+ end
108
+ end
109
+
110
+ #
111
+ # Unlock a specific job_id
112
+ #
113
+ # @param [String] job_id a sidekiq JID
114
+ #
115
+ # @return [true] when job_id was removed
116
+ # @return [false] when job_id wasn't locked
117
+ #
118
+ def unlock(job_id)
119
+ locked.del(job_id)
120
+ end
121
+
122
+ #
123
+ # Deletes all the redis keys for this lock
124
+ #
125
+ #
126
+ # @return [Integer] the number of keys deleted in redis
127
+ #
128
+ def del
129
+ redis do |conn|
130
+ conn.multi do |pipeline|
131
+ pipeline.zrem(DIGESTS, key.digest)
132
+ pipeline.del(key.digest, key.queued, key.primed, key.locked, key.info)
133
+ end
134
+ end
135
+ end
136
+
137
+ #
138
+ # Returns either the time the lock was initialized with or
139
+ # the first changelog entry's timestamp
140
+ #
141
+ #
142
+ # @return [Float] a floaty timestamp represantation
143
+ #
144
+ def created_at
145
+ @created_at ||= changelogs.first&.[]("time")
146
+ end
147
+
148
+ #
149
+ # Returns all job_id's for this lock
150
+ #
151
+ # @note a JID can be present in 3 different places
152
+ #
153
+ #
154
+ # @return [Array<String>] an array with JIDs
155
+ #
156
+ def all_jids
157
+ (queued_jids + primed_jids + locked_jids).uniq
158
+ end
159
+
160
+ #
161
+ # Returns a collection of locked job_id's
162
+ #
163
+ # @param [true, false] with_values false provide the timestamp for the lock
164
+ #
165
+ # @return [Hash<String, Float>] when given `with_values: true`
166
+ # @return [Array<String>] when given `with_values: false`
167
+ #
168
+ def locked_jids(with_values: false)
169
+ locked.entries(with_values: with_values)
170
+ end
171
+
172
+ #
173
+ # Returns the queued JIDs
174
+ #
175
+ #
176
+ # @return [Array<String>] an array with queued job_ids
177
+ #
178
+ def queued_jids
179
+ queued.entries
180
+ end
181
+
182
+ #
183
+ # Returns the primed JIDs
184
+ #
185
+ #
186
+ # @return [Array<String>] an array with primed job_ids
187
+ #
188
+ def primed_jids
189
+ primed.entries
190
+ end
191
+
192
+ #
193
+ # Returns all matching changelog entries for this lock
194
+ #
195
+ #
196
+ # @return [Array<Hash>] an array with changelogs
197
+ #
198
+ def changelogs
199
+ changelog.entries(pattern: "*#{key.digest}*")
200
+ end
201
+
202
+ #
203
+ # The digest key
204
+ #
205
+ # @note Used for exists checks to avoid enqueuing
206
+ # the same lock twice
207
+ #
208
+ #
209
+ # @return [Redis::String] a string representation of the key
210
+ #
211
+ def digest
212
+ @digest ||= Redis::String.new(key.digest)
213
+ end
214
+
215
+ #
216
+ # The queued list
217
+ #
218
+ #
219
+ # @return [Redis::List] for queued JIDs
220
+ #
221
+ def queued
222
+ @queued ||= Redis::List.new(key.queued)
223
+ end
224
+
225
+ #
226
+ # The primed list
227
+ #
228
+ #
229
+ # @return [Redis::List] for primed JIDs
230
+ #
231
+ def primed
232
+ @primed ||= Redis::List.new(key.primed)
233
+ end
234
+
235
+ #
236
+ # The locked hash
237
+ #
238
+ #
239
+ # @return [Redis::Hash] for locked JIDs
240
+ #
241
+ def locked
242
+ @locked ||= Redis::Hash.new(key.locked)
243
+ end
244
+
245
+ #
246
+ # Information about the lock
247
+ #
248
+ #
249
+ # @return [Redis::Hash] with lock information
250
+ #
251
+ def info
252
+ @info ||= LockInfo.new(key.info)
253
+ end
254
+
255
+ #
256
+ # A sorted set with changelog entries
257
+ #
258
+ # @see Changelog for more information
259
+ #
260
+ #
261
+ # @return [Changelog]
262
+ #
263
+ def changelog
264
+ @changelog ||= Changelog.new
265
+ end
266
+
267
+ #
268
+ # A nicely formatted string with information about this lock
269
+ #
270
+ #
271
+ # @return [String]
272
+ #
273
+ def to_s
274
+ <<~MESSAGE
275
+ Lock status for #{key}
276
+
277
+ value: #{digest.value}
278
+ info: #{info.value}
279
+ queued_jids: #{queued_jids}
280
+ primed_jids: #{primed_jids}
281
+ locked_jids: #{locked_jids}
282
+ changelogs: #{changelogs}
283
+ MESSAGE
284
+ end
285
+
286
+ #
287
+ # @see to_s
288
+ #
289
+ def inspect
290
+ to_s
291
+ end
292
+
293
+ private
294
+
295
+ #
296
+ # Ensure the key is a {Key}
297
+ #
298
+ # @param [String, Key] key
299
+ #
300
+ # @return [Key]
301
+ #
302
+ def get_key(key)
303
+ if key.is_a?(SidekiqUniqueJobs::Key)
304
+ key
305
+ else
306
+ SidekiqUniqueJobs::Key.new(key)
307
+ end
308
+ end
309
+
310
+ #
311
+ # Generate a changelog entry for the given arguments
312
+ #
313
+ # @param [String] job_id a sidekiq JID
314
+ # @param [String] script the name of the script generating this entry
315
+ # @param [String] message a descriptive message for later review
316
+ #
317
+ # @return [String] a JSON string matching the Lua script structure
318
+ #
319
+ def changelog_json(job_id, script, message)
320
+ dump_json(
321
+ digest: key.digest,
322
+ job_id: job_id,
323
+ script: script,
324
+ message: message,
325
+ time: now_f,
326
+ )
327
+ end
328
+
329
+ #
330
+ # Add the digest to the correct sorted set
331
+ #
332
+ # @param [Object] pipeline a redis pipeline object for issue commands
333
+ # @param [Hash] lock_info the lock info relevant to the digest
334
+ #
335
+ # @return [nil]
336
+ #
337
+ def add_digest_to_set(pipeline, lock_info, score = nil)
338
+ score ||= now_f
339
+ digest_string = key.digest
340
+
341
+ if lock_info["lock"] == :until_expired
342
+ pipeline.zadd(key.expiring_digests, score + lock_info["ttl"], digest_string)
343
+ else
344
+ pipeline.zadd(key.digests, score, digest_string)
345
+ end
346
+ end
347
+ end
348
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Handles uniqueness of sidekiq arguments
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class LockArgs
8
+ include SidekiqUniqueJobs::Logging
9
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
10
+ include SidekiqUniqueJobs::JSON
11
+
12
+ # Convenience method for returning a digest
13
+ # @param [Hash] item a Sidekiq job hash
14
+ # @return [String] a unique digest
15
+ def self.call(item)
16
+ new(item).lock_args
17
+ end
18
+
19
+ # The sidekiq job hash
20
+ # @return [Hash] the Sidekiq job hash
21
+ attr_reader :item
22
+ #
23
+ # @!attribute [r] args
24
+ # @return [Array<Objet>] the arguments passed to `perform_async`
25
+ attr_reader :args
26
+
27
+ # @param [Hash] item a Sidekiq job hash
28
+ def initialize(item)
29
+ @item = item
30
+ @args = item[ARGS]
31
+ self.job_class = item[CLASS]
32
+ end
33
+
34
+ # The unique arguments to use for creating a lock
35
+ # @return [Array] the arguments filters by the {#filtered_args} method if {#lock_args_enabled?}
36
+ def lock_args
37
+ @lock_args ||= filtered_args || []
38
+ end
39
+
40
+ # Checks if the worker class has enabled lock_args
41
+ # @return [true, false]
42
+ def lock_args_enabled?
43
+ # return false unless lock_args_method_valid?
44
+
45
+ lock_args_method
46
+ end
47
+
48
+ # Validate that the lock_args_method is acceptable
49
+ # @return [true, false]
50
+ def lock_args_method_valid?
51
+ [NilClass, TrueClass, FalseClass].none? { |klass| lock_args_method.is_a?(klass) }
52
+ end
53
+
54
+ # Checks if the worker class has disabled lock_args
55
+ # @return [true, false]
56
+ def lock_args_disabled?
57
+ !lock_args_method
58
+ end
59
+
60
+ # Filters unique arguments by proc or symbol
61
+ # @return [Array] {#filter_by_proc} when {#lock_args_method} is a Proc
62
+ # @return [Array] {#filter_by_symbol} when {#lock_args_method} is a Symbol
63
+ # @return [Array] args unfiltered when neither of the above
64
+ def filtered_args
65
+ return args if lock_args_disabled?
66
+
67
+ json_args = Normalizer.jsonify(args)
68
+
69
+ case lock_args_method
70
+ when Proc
71
+ filter_by_proc(json_args)
72
+ when Symbol
73
+ filter_by_symbol(json_args)
74
+ end
75
+ end
76
+
77
+ # Filters unique arguments by proc configured in the sidekiq worker
78
+ # @param [Array] args the arguments passed to the sidekiq worker
79
+ # @return [Array] with the filtered arguments
80
+ def filter_by_proc(args)
81
+ lock_args_method.call(args)
82
+ end
83
+
84
+ # Filters unique arguments by method configured in the sidekiq worker
85
+ # @param [Array] args the arguments passed to the sidekiq worker
86
+ # @return [Array] unfiltered unless {#job_method_defined?}
87
+ # @return [Array] with the filtered arguments
88
+ def filter_by_symbol(args)
89
+ return args unless job_method_defined?(lock_args_method)
90
+
91
+ job_class.send(lock_args_method, args)
92
+ rescue ArgumentError
93
+ raise SidekiqUniqueJobs::InvalidUniqueArguments,
94
+ given: args,
95
+ job_class: job_class,
96
+ lock_args_method: lock_args_method
97
+ end
98
+
99
+ # The method to use for filtering unique arguments
100
+ def lock_args_method
101
+ @lock_args_method ||= job_options.slice(LOCK_ARGS_METHOD, UNIQUE_ARGS_METHOD).values.first
102
+ @lock_args_method ||= :lock_args if job_method_defined?(:lock_args)
103
+ @lock_args_method ||= :unique_args if job_method_defined?(:unique_args)
104
+ @lock_args_method ||= default_lock_args_method
105
+ end
106
+
107
+ # The global worker options defined in Sidekiq directly
108
+ def default_lock_args_method
109
+ default_job_options[LOCK_ARGS_METHOD] ||
110
+ default_job_options[UNIQUE_ARGS_METHOD]
111
+ end
112
+
113
+ #
114
+ # The globally default worker options configured from Sidekiq
115
+ #
116
+ #
117
+ # @return [Hash<String, Object>]
118
+ #
119
+ def default_job_options
120
+ @default_job_options ||= if Sidekiq.respond_to?(:default_job_options)
121
+ Sidekiq.default_job_options.stringify_keys
122
+ else
123
+ Sidekiq.default_worker_options.stringify_keys
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Gathers all configuration for a lock
6
+ # which helps reduce the amount of instance variables
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class LockConfig
11
+ #
12
+ # @!attribute [r] type
13
+ # @return [Symbol] the type of lock
14
+ attr_reader :type
15
+ #
16
+ # @!attribute [r] job
17
+ # @return [Symbol] the job class
18
+ attr_reader :job
19
+ #
20
+ # @!attribute [r] limit
21
+ # @return [Integer] the number of simultaneous locks
22
+ attr_reader :limit
23
+ #
24
+ # @!attribute [r] timeout
25
+ # @return [Integer, nil] the time to wait for a lock
26
+ attr_reader :timeout
27
+ #
28
+ # @!attribute [r] ttl
29
+ # @return [Integer, nil] the time (in seconds) to live after successful
30
+ attr_reader :ttl
31
+ #
32
+ # @!attribute [r] ttl
33
+ # @return [Integer, nil] the time (in milliseconds) to live after successful
34
+ attr_reader :pttl
35
+ #
36
+ # @!attribute [r] lock_info
37
+ # @return [Boolean] indicate wether to use lock_info or not
38
+ attr_reader :lock_info
39
+ #
40
+ # @!attribute [r] on_conflict
41
+ # @return [Symbol, Hash<Symbol, Symbol>] the strategies to use as conflict resolution
42
+ attr_reader :on_conflict
43
+ #
44
+ # @!attribute [r] errors
45
+ # @return [Array<Hash<Symbol, Array<String>] a collection of configuration errors
46
+ attr_reader :errors
47
+
48
+ #
49
+ # Instantiate a new lock_config based on sidekiq options in worker
50
+ #
51
+ # @param [Hash] options sidekiq_options for worker
52
+ #
53
+ # @return [LockConfig]
54
+ #
55
+ def self.from_worker(options)
56
+ new(options.deep_stringify_keys)
57
+ end
58
+
59
+ def initialize(job_hash = {})
60
+ @type = job_hash[LOCK]&.to_sym
61
+ @job = SidekiqUniqueJobs.safe_constantize(job_hash[CLASS])
62
+ @limit = job_hash.fetch(LOCK_LIMIT, 1)&.to_i
63
+ @timeout = job_hash.fetch(LOCK_TIMEOUT, 0)&.to_i
64
+ @ttl = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION, nil) }.to_i
65
+ @pttl = ttl * 1_000
66
+ @lock_info = job_hash.fetch(LOCK_INFO) { SidekiqUniqueJobs.config.lock_info }
67
+ @on_conflict = job_hash.fetch(ON_CONFLICT, nil)
68
+ @errors = job_hash.fetch(ERRORS) { {} }
69
+
70
+ @on_client_conflict = job_hash[ON_CLIENT_CONFLICT]
71
+ @on_server_conflict = job_hash[ON_SERVER_CONFLICT]
72
+ end
73
+
74
+ def lock_info?
75
+ lock_info
76
+ end
77
+
78
+ #
79
+ # Indicate if timeout was set
80
+ #
81
+ #
82
+ # @return [true,false]
83
+ #
84
+ def wait_for_lock?
85
+ timeout.nil? || timeout.positive?
86
+ end
87
+
88
+ #
89
+ # Is the configuration valid?
90
+ #
91
+ #
92
+ # @return [true,false]
93
+ #
94
+ def valid?
95
+ errors.empty?
96
+ end
97
+
98
+ #
99
+ # Return a nice descriptive message with all validation errors
100
+ #
101
+ #
102
+ # @return [String]
103
+ #
104
+ def errors_as_string
105
+ return if valid?
106
+
107
+ @errors_as_string ||= begin
108
+ error_msg = +"\t"
109
+ error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")
110
+ error_msg
111
+ end
112
+ end
113
+
114
+ # the strategy to use as conflict resolution from sidekiq client
115
+ def on_client_conflict
116
+ @on_client_conflict ||= if on_conflict.is_a?(Hash)
117
+ on_conflict["client"] || on_conflict[:client]
118
+ else
119
+ on_conflict
120
+ end
121
+ end
122
+
123
+ # the strategy to use as conflict resolution from sidekiq server
124
+ def on_server_conflict
125
+ @on_server_conflict ||= if on_conflict.is_a?(Hash)
126
+ on_conflict["server"] || on_conflict[:server]
127
+ else
128
+ on_conflict
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "openssl"
4
+
5
+ module SidekiqUniqueJobs
6
+ # Handles uniqueness of sidekiq arguments
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ class LockDigest
10
+ include SidekiqUniqueJobs::Logging
11
+ include SidekiqUniqueJobs::JSON
12
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
13
+
14
+ #
15
+ # Generates a new digest
16
+ #
17
+ # @param [Hash] item a sidekiq job hash
18
+ #
19
+ # @return [String] a unique digest for the given arguments
20
+ #
21
+ def self.call(item)
22
+ new(item).lock_digest
23
+ end
24
+
25
+ # The sidekiq job hash
26
+ # @return [Hash] the Sidekiq job hash
27
+ attr_reader :item
28
+ #
29
+ # @!attribute [r] args
30
+ # @return [Array<Objet>] the arguments passed to `perform_async`
31
+ attr_reader :lock_args
32
+ #
33
+ # @!attribute [r] args
34
+ # @return [String] the prefix for the unique key
35
+ attr_reader :lock_prefix
36
+
37
+ # @param [Hash] item a Sidekiq job hash
38
+ def initialize(item)
39
+ @item = item
40
+ @lock_args = item[LOCK_ARGS] || item[UNIQUE_ARGS] # TODO: Deprecate UNIQUE_ARGS
41
+ @lock_prefix = item[LOCK_PREFIX] || item[UNIQUE_PREFIX] # TODO: Deprecate UNIQUE_PREFIX
42
+ self.job_class = item[CLASS]
43
+ end
44
+
45
+ # Memoized lock_digest
46
+ # @return [String] a unique digest
47
+ def lock_digest
48
+ @lock_digest ||= create_digest
49
+ end
50
+
51
+ # Creates a namespaced unique digest based on the {#digestable_hash} and the {#lock_prefix}
52
+ # @return [String] a unique digest
53
+ def create_digest
54
+ digest = OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort))
55
+ "#{lock_prefix}:#{digest}"
56
+ end
57
+
58
+ # Filter a hash to use for digest
59
+ # @return [Hash] to use for digest
60
+ def digestable_hash
61
+ @item.slice(CLASS, QUEUE, LOCK_ARGS, APARTMENT).tap do |hash|
62
+ hash.delete(QUEUE) if unique_across_queues?
63
+ hash.delete(CLASS) if unique_across_workers?
64
+ end
65
+ end
66
+
67
+ # Checks if we should disregard the queue when creating the unique digest
68
+ # @return [true, false]
69
+ def unique_across_queues?
70
+ item[UNIQUE_ACROSS_QUEUES] || job_options[UNIQUE_ACROSS_QUEUES]
71
+ end
72
+
73
+ # Checks if we should disregard the worker when creating the unique digest
74
+ # @return [true, false]
75
+ def unique_across_workers?
76
+ item[UNIQUE_ACROSS_WORKERS] || job_options[UNIQUE_ACROSS_WORKERS]
77
+ end
78
+ end
79
+ end