sidekiq-unique-jobs 6.0.6 → 6.0.25

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +768 -150
  3. data/README.md +123 -77
  4. data/bin/uniquejobs +2 -2
  5. data/lib/sidekiq-unique-jobs.rb +1 -1
  6. data/lib/sidekiq_unique_jobs/cli.rb +27 -12
  7. data/lib/sidekiq_unique_jobs/client/middleware.rb +8 -4
  8. data/lib/sidekiq_unique_jobs/constants.rb +29 -18
  9. data/lib/sidekiq_unique_jobs/digests.rb +34 -19
  10. data/lib/sidekiq_unique_jobs/job.rb +29 -0
  11. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +14 -18
  12. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +20 -4
  13. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +7 -3
  14. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
  15. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +1 -0
  16. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +9 -3
  17. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +0 -8
  18. data/lib/sidekiq_unique_jobs/locksmith.rb +59 -32
  19. data/lib/sidekiq_unique_jobs/logging.rb +14 -0
  20. data/lib/sidekiq_unique_jobs/middleware.rb +26 -9
  21. data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
  22. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  23. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +3 -3
  24. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +7 -2
  25. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
  26. data/lib/sidekiq_unique_jobs/on_conflict.rb +12 -7
  27. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +8 -8
  28. data/lib/sidekiq_unique_jobs/scripts.rb +38 -9
  29. data/lib/sidekiq_unique_jobs/server/middleware.rb +18 -6
  30. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +1 -1
  31. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +88 -0
  32. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +13 -4
  33. data/lib/sidekiq_unique_jobs/testing.rb +3 -3
  34. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +2 -1
  35. data/lib/sidekiq_unique_jobs/timeout.rb +1 -1
  36. data/lib/sidekiq_unique_jobs/unique_args.rb +7 -4
  37. data/lib/sidekiq_unique_jobs/util.rb +8 -4
  38. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  39. data/lib/sidekiq_unique_jobs/version_check.rb +95 -0
  40. data/lib/sidekiq_unique_jobs/web/helpers.rb +7 -6
  41. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +4 -0
  42. data/lib/sidekiq_unique_jobs/web.rb +19 -12
  43. data/lib/sidekiq_unique_jobs.rb +33 -107
  44. data/lib/tasks/changelog.rake +23 -0
  45. data/redis/convert_legacy_lock.lua +13 -0
  46. data/redis/delete_by_digest.lua +10 -11
  47. data/redis/delete_job_by_digest.lua +6 -4
  48. data/redis/lock.lua +14 -10
  49. data/redis/unlock.lua +13 -11
  50. metadata +88 -108
  51. data/.codeclimate.yml +0 -35
  52. data/.csslintrc +0 -2
  53. data/.dockerignore +0 -4
  54. data/.editorconfig +0 -14
  55. data/.eslintignore +0 -1
  56. data/.eslintrc +0 -213
  57. data/.fasterer.yml +0 -23
  58. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -31
  59. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -17
  60. data/.gitignore +0 -26
  61. data/.reek.yml +0 -87
  62. data/.rspec +0 -2
  63. data/.rubocop.yml +0 -127
  64. data/.simplecov +0 -20
  65. data/.travis.yml +0 -49
  66. data/.yardopts +0 -7
  67. data/Appraisals +0 -25
  68. data/CODE_OF_CONDUCT.md +0 -74
  69. data/Gemfile +0 -24
  70. data/Guardfile +0 -55
  71. data/Rakefile +0 -12
  72. data/_config.yml +0 -1
  73. data/assets/unique_digests_1.png +0 -0
  74. data/assets/unique_digests_2.png +0 -0
  75. data/bin/bench +0 -20
  76. data/examples/another_unique_job.rb +0 -15
  77. data/examples/custom_queue_job.rb +0 -12
  78. data/examples/custom_queue_job_with_filter_method.rb +0 -13
  79. data/examples/custom_queue_job_with_filter_proc.rb +0 -16
  80. data/examples/expiring_job.rb +0 -12
  81. data/examples/inline_worker.rb +0 -12
  82. data/examples/just_a_worker.rb +0 -13
  83. data/examples/long_running_job.rb +0 -14
  84. data/examples/main_job.rb +0 -14
  85. data/examples/my_job.rb +0 -12
  86. data/examples/my_unique_job.rb +0 -15
  87. data/examples/my_unique_job_with_filter_method.rb +0 -21
  88. data/examples/my_unique_job_with_filter_proc.rb +0 -19
  89. data/examples/notify_worker.rb +0 -14
  90. data/examples/plain_class.rb +0 -13
  91. data/examples/simple_worker.rb +0 -15
  92. data/examples/spawn_simple_worker.rb +0 -12
  93. data/examples/test_class.rb +0 -9
  94. data/examples/unique_across_workers_job.rb +0 -20
  95. data/examples/unique_job_on_conflict_raise.rb +0 -14
  96. data/examples/unique_job_on_conflict_reject.rb +0 -14
  97. data/examples/unique_job_on_conflict_reschedule.rb +0 -14
  98. data/examples/unique_job_with_conditional_parameter.rb +0 -18
  99. data/examples/unique_job_with_filter_method.rb +0 -21
  100. data/examples/unique_job_with_nil_unique_args.rb +0 -20
  101. data/examples/unique_job_with_no_unique_args_method.rb +0 -16
  102. data/examples/unique_job_withthout_unique_args_parameter.rb +0 -18
  103. data/examples/unique_on_all_queues_job.rb +0 -16
  104. data/examples/until_and_while_executing_job.rb +0 -17
  105. data/examples/until_executed_2_job.rb +0 -24
  106. data/examples/until_executed_job.rb +0 -25
  107. data/examples/until_executing_job.rb +0 -11
  108. data/examples/until_expired_job.rb +0 -12
  109. data/examples/until_global_expired_job.rb +0 -12
  110. data/examples/while_executing_job.rb +0 -15
  111. data/examples/while_executing_reject_job.rb +0 -14
  112. data/examples/without_argument_job.rb +0 -13
  113. data/sidekiq-unique-jobs.gemspec +0 -42
data/README.md CHANGED
@@ -1,43 +1,49 @@
1
- # SidekiqUniqueJobs [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/mhenrixon/sidekiq-unique-jobs.png?branch=master)](https://travis-ci.org/mhenrixon/sidekiq-unique-jobs) [![Code Climate](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs.png)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs) [![Test Coverage](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/badges/coverage.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/coverage)
2
-
3
- ## Table of contents
4
-
5
- * [Introduction](#introduction)
6
- * [Documentation](#documentation)
7
- * [Requirements](#requirements)
8
- * [Installation](#installation)
9
- * [Support Me](#support-me)
10
- * [General Information](#general-information)
11
- * [Options](#options)
12
- * [Lock Expiration](#lock-expiration)
13
- * [Lock Timeout](#lock-timeout)
14
- * [Unique Across Queues](#unique-across-queues)
15
- * [Unique Across Workers](#unique-across-workers)
16
- * [Locks](#locks)
17
- * [Until Executing](#until-executing)
18
- * [Until Executed](#until-executed)
19
- * [Until Timeout](#until-timeout)
20
- * [Unique Until And While Executing](#unique-until-and-while-executing)
21
- * [While Executing](#while-executing)
22
- * [Conflict Strategy](#conflict-strategy)
23
- * [Log](#log)
24
- * [Raise](#raise)
25
- * [Reject](#reject)
26
- * [Replace](#replace)
27
- * [Reschedule](#reschedule)
28
- * [Usage](#usage)
29
- * [Finer Control over Uniqueness](#finer-control-over-uniqueness)
30
- * [After Unlock Callback](#after-unlock-callback)
31
- * [Logging](#logging)
32
- * [Cleanup Dead Locks](#cleanup-dead-locks)
33
- * [Debugging](#debugging)
34
- * [Sidekiq Web](#sidekiq-web)
35
- * [Show Unique Digests](#show-unique-digests)
36
- * [Show keys for digest](#show-keys-for-digest)
37
- * [Communication](#communication)
38
- * [Testing](#testing)
39
- * [Contributing](#contributing)
40
- * [Contributors](#contributors)
1
+ # SidekiqUniqueJobs [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.com/mhenrixon/sidekiq-unique-jobs.svg?branch=v6.x)](https://travis-ci.com/mhenrixon/sidekiq-unique-jobs) [![Code Climate](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs.png)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs) [![Test Coverage](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/badges/coverage.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/coverage)
2
+
3
+ <!-- MarkdownTOC -->
4
+
5
+ - [Introduction](#introduction)
6
+ - [Documentation](#documentation)
7
+ - [Requirements](#requirements)
8
+ - [ActiveJob](#activejob)
9
+ - [redis-namespace](#redis-namespace)
10
+ - [Installation](#installation)
11
+ - [Support Me](#support-me)
12
+ - [General Information](#general-information)
13
+ - [Options](#options)
14
+ - [Lock Expiration](#lock-expiration)
15
+ - [Lock Timeout](#lock-timeout)
16
+ - [Unique Across Queues](#unique-across-queues)
17
+ - [Unique Across Workers](#unique-across-workers)
18
+ - [Locks](#locks)
19
+ - [Until Executing](#until-executing)
20
+ - [Until Executed](#until-executed)
21
+ - [Until Timeout](#until-timeout)
22
+ - [Unique Until And While Executing](#unique-until-and-while-executing)
23
+ - [While Executing](#while-executing)
24
+ - [Conflict Strategy](#conflict-strategy)
25
+ - [Log](#log)
26
+ - [Raise](#raise)
27
+ - [Reject](#reject)
28
+ - [Replace](#replace)
29
+ - [Reschedule](#reschedule)
30
+ - [Usage](#usage)
31
+ - [Finer Control over Uniqueness](#finer-control-over-uniqueness)
32
+ - [After Unlock Callback](#after-unlock-callback)
33
+ - [Logging](#logging)
34
+ - [Cleanup Dead Locks](#cleanup-dead-locks)
35
+ - [Other Sidekiq gems](#other-sidekiq-gems)
36
+ - [sidekiq-global_id](#sidekiq-global_id)
37
+ - [Debugging](#debugging)
38
+ - [Sidekiq Web](#sidekiq-web)
39
+ - [Show Unique Digests](#show-unique-digests)
40
+ - [Show keys for digest](#show-keys-for-digest)
41
+ - [Communication](#communication)
42
+ - [Testing](#testing)
43
+ - [Contributing](#contributing)
44
+ - [Contributors](#contributors)
45
+
46
+ <!-- /MarkdownTOC -->
41
47
 
42
48
  ## Introduction
43
49
 
@@ -45,37 +51,50 @@ The goal of this gem is to ensure your Sidekiq jobs are unique. We do this by cr
45
51
 
46
52
  ## Documentation
47
53
 
48
- This is the documentation for the master branch. You can find the documentation for each release by navigating to its tag: https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v5.0.10.
54
+ This is the documentation for the master branch. You can find the documentation for each release by navigating to its tag: [v5.0.10][]
49
55
 
50
56
  Below are links to the latest major versions (4 & 5):
51
57
 
52
- - [v5.0.10](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v5.0.10)
53
- - [v4.0.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v4.0.18)
58
+ - [v5.0.10][]
59
+ - [v4.0.18][]
54
60
 
55
61
  ## Requirements
56
62
 
57
- See https://github.com/mperham/sidekiq#requirements for what is required. Starting from 5.0.0 only sidekiq >= 4 is supported and support for MRI <= 2.1 is dropped. ActiveJob is not supported
63
+ See [Sidekiq requirements][] for what is required. Starting from 5.0.0 only sidekiq >= 4 and MRI >= 2.2. ActiveJob is not supported
58
64
 
59
- Version 6 requires Redis >= 3 and pure Sidekiq, no ActiveJob supported anymore. See [About ActiveJob](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/About-ActiveJob) for why.
65
+ ### ActiveJob
66
+
67
+ Version 6 requires Redis >= 3 and pure Sidekiq, no ActiveJob supported anymore. See [About ActiveJob](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/About-ActiveJob) for why. It simply is too complex and generates more issues than I can handle given how little timer I have to spend on this project.
68
+
69
+ ### redis-namespace
70
+
71
+ Will not be officially supported anymore. Since Mike [won't support redis-namespace](https://github.com/mperham/sidekiq/issues/3366#issuecomment-284270120) neither will I.
72
+
73
+ [Read this](http://www.mikeperham.com/2017/04/10/migrating-from-redis-namespace/) for how to migrate away from namespacing.
60
74
 
61
75
  ## Installation
62
76
 
63
77
  Add this line to your application's Gemfile:
64
78
 
65
- gem 'sidekiq-unique-jobs'
79
+ ```
80
+ gem 'sidekiq-unique-jobs'
81
+ ```
66
82
 
67
83
  And then execute:
68
84
 
69
- $ bundle
85
+ ```
86
+ bundle
87
+ ```
70
88
 
71
89
  Or install it yourself as:
72
90
 
73
- $ gem install sidekiq-unique-jobs
74
-
91
+ ```
92
+ gem install sidekiq-unique-jobs
93
+ ```
75
94
 
76
95
  ## Support Me
77
96
 
78
- Want to show me some ❤️ for the hard work I do on this gem? You can use the following PayPal link https://paypal.me/mhenrixon. Any amount is welcome and let me tell you it feels good to be appreciated. Even a dollar makes me super excited about all of this.
97
+ Want to show me some ❤️ for the hard work I do on this gem? You can use the following [PayPal link][]. Any amount is welcome and let me tell you it feels good to be appreciated. Even a dollar makes me super excited about all of this.
79
98
 
80
99
  ## General Information
81
100
 
@@ -87,15 +106,13 @@ See [Locking & Unlocking](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/
87
106
 
88
107
  ### Lock Expiration
89
108
 
90
- This is probably not the configuration option you want...
91
-
92
- Since the client and the server are disconnected and not running inside the same process, setting a lock expiration is probably not what you want. Any keys that are used by this gem WILL be removed at the time of the expiration. For jobs that are scheduled in the future the key will expire when that job is scheduled + whatever expiration you have set.
109
+ Lock expiration is used for two things. For the `UntilExpired` job releases the lock upon expiry. This is done from the client.
93
110
 
94
- In previous versions there was a default expiration of 30 minutes which didn't work for a lot of long running jobs. Since version 6 there will be no expiration of any jobs from the default configuration. Please don't use `lock_expiration` unless you really know what you are doing.
111
+ Since v6.0.11 the other locks will expire after the server is done processing.
95
112
 
96
113
  ```ruby
97
114
  sidekiq_options lock_expiration: nil # default - don't expire keys
98
- sidekiq_options lock_expiration: 20.days # expire this lock in 20 days
115
+ sidekiq_options lock_expiration: 20.days.to_i # expire this lock in 20 days
99
116
  ```
100
117
 
101
118
  ### Lock Timeout
@@ -115,7 +132,7 @@ This configuration option is slightly misleading. It doesn't disregard the queue
115
132
  ```ruby
116
133
  class Worker
117
134
  include Sidekiq::Worker
118
-
135
+
119
136
  sidekiq_options unique_across_queues: true, queue: 'default'
120
137
 
121
138
  def perform(args); end
@@ -131,7 +148,7 @@ This configuration option is slightly misleading. It doesn't disregard the worke
131
148
  ```ruby
132
149
  class WorkerOne
133
150
  include Sidekiq::Worker
134
-
151
+
135
152
  sidekiq_options unique_across_workers: true, queue: 'default'
136
153
 
137
154
  def perform(args); end
@@ -139,17 +156,17 @@ end
139
156
 
140
157
  class WorkerTwo
141
158
  include Sidekiq::Worker
142
-
159
+
143
160
  sidekiq_options unique_across_workers: true, queue: 'default'
144
161
 
145
162
  def perform(args); end
146
163
  end
147
164
 
148
165
 
149
- WorkerOne.perform_async(1)
166
+ WorkerOne.perform_async(1)
150
167
  # => 'the jobs unique id'
151
168
 
152
- WorkerTwo.perform_async(1)
169
+ WorkerTwo.perform_async(1)
153
170
  # => nil because WorkerOne just stole the lock
154
171
  ```
155
172
 
@@ -220,7 +237,7 @@ In the console you should see something like:
220
237
 
221
238
  ## Conflict Strategy
222
239
 
223
- Decides how we handle conflict. We can either reject the job to the dead queue or reschedule it. Both are useful for jobs that absolutely need to run and have been configured to use the lock `WhileExecuting` that is used only by the sidekiq server process.
240
+ Decides how we handle conflict. We can either reject the job to the dead queue or reschedule it. Both are useful for jobs that absolutely need to run and have been configured to use the lock `WhileExecuting` that is used only by the sidekiq server process.
224
241
 
225
242
  The last one is log which can be be used with the lock `UntilExecuted` and `UntilExpired`. Now we write a log entry saying the job could not be pushed because it is a duplicate of another job with the same arguments
226
243
 
@@ -246,9 +263,9 @@ This strategy is intended to be used with `WhileExecuting` and will push the job
246
263
 
247
264
  This strategy is intended to be used with client locks like `UntilExecuted`.
248
265
  It will delete any existing job for these arguments from retry, schedule and
249
- queue and retry the lock again.
266
+ queue and retry the lock again.
250
267
 
251
- This is slightly dangerous and should probably only be used for jobs that are
268
+ This is slightly dangerous and should probably only be used for jobs that are
252
269
  always scheduled in the future. Currently only attempting to retry one time.
253
270
 
254
271
  `sidekiq_options lock: :until_executed, on_conflict: :replace`
@@ -322,16 +339,22 @@ end
322
339
 
323
340
  ### After Unlock Callback
324
341
 
325
- If you need to perform any additional work after the lock has been released you can provide an `#after_unlock` instance method. The method will be called when the lock has been unlocked. Most times this means after yield but there are two exceptions to that.
342
+ If you need to perform any additional work after the lock has been released you can provide an `#after_unlock` instance method. The method will be called when the lock has been unlocked. Most times this means after yield but there are two exceptions to that.
326
343
 
327
344
  **Exception 1:** UntilExecuting unlocks and calls back before yielding.
328
345
  **Exception 2:** UntilExpired expires eventually, no after_unlock hook is called.
329
346
 
347
+ **NOTE:** _It is also possible to write this code as a class method._
348
+
330
349
  ```ruby
331
350
  class UniqueJobWithFilterMethod
332
351
  include Sidekiq::Worker
333
352
  sidekiq_options lock: :while_executing,
334
353
 
354
+ def self.after_unlock
355
+ # block has yielded and lock is released
356
+ end
357
+
335
358
  def after_unlock
336
359
  # block has yielded and lock is released
337
360
  end
@@ -356,12 +379,12 @@ end
356
379
 
357
380
  ### Cleanup Dead Locks
358
381
 
359
- For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class.
382
+ For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class. This is deprecated in Sidekiq 6.0
360
383
 
361
384
  ```ruby
362
385
  class MyWorker
363
386
  sidekiq_retries_exhausted do |msg, _ex|
364
- SidekiqUniqueJobs::Digests.del(digest: msg['unique_digest']) if msg['unique_digest']
387
+ SidekiqUniqueJobs::Digests.delete_by_digest(msg['unique_digest']) if msg['unique_digest']
365
388
  end
366
389
  end
367
390
  ```
@@ -372,11 +395,29 @@ Starting in v5.1, Sidekiq can also fire a global callback when a job dies:
372
395
  # this goes in your initializer
373
396
  Sidekiq.configure_server do |config|
374
397
  config.death_handlers << ->(job, _ex) do
375
- SidekiqUniqueJobs::Digests.del(digest: job['unique_digest']) if job['unique_digest']
398
+ SidekiqUniqueJobs::Digests.delete_by_digest(job['unique_digest']) if job['unique_digest']
376
399
  end
377
400
  end
378
401
  ```
379
402
 
403
+ ### Other Sidekiq gems
404
+
405
+ #### sidekiq-global_id
406
+
407
+ It was reported in [#235](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/235) that the order of the Sidekiq middleware needs to be as follows.
408
+
409
+ ```ruby
410
+ Sidekiq.client_middleware do |chain|
411
+ chain.add Sidekiq::GlobalId::ClientMiddleware
412
+ chain.add SidekiqUniqueJobs::Client::Middleware
413
+ end
414
+
415
+ Sidekiq.server_middleware do |chain|
416
+ chain.add SidekiqUniqueJobs::Server::Middleware
417
+ chain.add Sidekiq::GlobalId::ServerMiddleware
418
+ end
419
+ ```
420
+
380
421
  ## Debugging
381
422
 
382
423
  There are several ways of removing keys that are stuck. The prefered way is by using the unique extension to `Sidekiq::Web`. The old console and command line versions still work but might be deprecated in the future. It is better to search for the digest itself and delete the keys matching that digest.
@@ -391,12 +432,11 @@ require 'sidekiq_unique_jobs/web'
391
432
  mount Sidekiq::Web, at: '/sidekiq'
392
433
  ```
393
434
 
394
- There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web`
435
+ There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web`
395
436
  already does this.
396
437
 
397
438
  To filter/search for keys we can use the wildcard `*`. If we have a unique digest `'uniquejobs:9e9b5ce5d423d3ea470977004b50ff84` we can search for it by enter `*ff84` and it should return all digests that end with `ff84`.
398
439
 
399
-
400
440
  #### Show Unique Digests
401
441
 
402
442
  ![Unique Digests](assets/unique_digests_1.png)
@@ -413,7 +453,7 @@ There is a [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](
413
453
 
414
454
  This has been probably the most confusing part of this gem. People get really confused with how unreliable the unique jobs have been. I there for decided to do what Mike is doing for sidekiq enterprise. Read the section about unique jobs.
415
455
 
416
- https://www.dailydrip.com/topics/sidekiq/drips/sidekiq-enterprise-unique-jobs
456
+ [Enterprise unique jobs][]
417
457
 
418
458
  ```ruby
419
459
  SidekiqUniqueJobs.configure do |config|
@@ -443,7 +483,7 @@ RSpec.describe Workers::CoolOne do
443
483
 
444
484
  it 'prevents duplicate jobs from being scheduled' do
445
485
  SidekiqUniqueJobs.use_config(enabled: true) do
446
- expect(described_class.perform_async(1)).not_to eq(nil)
486
+ expect(described_class.perform_in(3600, 1)).not_to eq(nil)
447
487
  expect(described_class.perform_async(1)).to eq(nil)
448
488
  end
449
489
  end
@@ -462,12 +502,18 @@ I would strongly suggest you let this gem test uniqueness. If you care about how
462
502
  ## Contributing
463
503
 
464
504
  1. Fork it
465
- 2. Create your feature branch (`git checkout -b my-new-feature`)
466
- 3. Commit your changes (`git commit -am 'Add some feature'`)
467
- 4. Push to the branch (`git push origin my-new-feature`)
468
- 5. Create new Pull Request
505
+ 1. Create your feature branch (`git checkout -b my-new-feature`)
506
+ 1. Commit your changes (`git commit -am 'Add some feature'`)
507
+ 1. Push to the branch (`git push origin my-new-feature`)
508
+ 1. Create new Pull Request
469
509
 
510
+ ## Contributors
470
511
 
471
- ## Contributors
512
+ You can find a list of contributors over on [Contributors][]
472
513
 
473
- You can find a list of contributors over on https://github.com/mhenrixon/sidekiq-unique-jobs/graphs/contributors
514
+ [v5.0.10]: https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v5.0.10.
515
+ [v4.0.18]: https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v4.0.18
516
+ [Sidekiq requirements]: https://github.com/mperham/sidekiq#requirements
517
+ [Enterprise unique jobs]: https://www.dailydrip.com/topics/sidekiq/drips/sidekiq-enterprise-unique-jobs
518
+ [Contributors]: https://github.com/mhenrixon/sidekiq-unique-jobs/graphs/contributors
519
+ [Paypal link https://paypal.me/mhenrixon]: https://paypal.me/mhenrixon
data/bin/uniquejobs CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- require 'bundler/setup'
5
- require 'sidekiq_unique_jobs'
4
+ require "bundler/setup"
5
+ require "sidekiq_unique_jobs"
6
6
 
7
7
  SidekiqUniqueJobs::Cli.start(ARGV)
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq_unique_jobs'
3
+ require "sidekiq_unique_jobs"
@@ -1,24 +1,29 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'thor'
3
+ require "thor"
4
4
 
5
5
  module SidekiqUniqueJobs
6
+ #
7
+ # Command line interface for unique jobs
8
+ #
9
+ # @author Mikael Henriksson <mikael@zoolutions.se>
10
+ #
6
11
  class Cli < Thor
7
12
  def self.banner(command, _namespace = nil, _subcommand = false)
8
13
  "jobs #{@package_name} #{command.usage}"
9
14
  end
10
15
 
11
- desc 'keys PATTERN', 'list all unique keys and their expiry time'
12
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
13
- def keys(pattern = '*')
16
+ desc "keys PATTERN", "list all unique keys and their expiry time"
17
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
18
+ def keys(pattern = "*")
14
19
  keys = Util.keys(pattern, options[:count])
15
20
  say "Found #{keys.size} keys matching '#{pattern}':"
16
21
  print_in_columns(keys.sort) if keys.any?
17
22
  end
18
23
 
19
- desc 'del PATTERN', 'deletes unique keys from redis by pattern'
20
- option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
21
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
24
+ desc "del PATTERN", "deletes unique keys from redis by pattern"
25
+ option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
26
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
22
27
  def del(pattern)
23
28
  max_count = options[:count]
24
29
  if options[:dry_run]
@@ -30,7 +35,7 @@ module SidekiqUniqueJobs
30
35
  end
31
36
  end
32
37
 
33
- desc 'console', 'drop into a console with easy access to helper methods'
38
+ desc "console", "drop into a console with easy access to helper methods"
34
39
  def console
35
40
  say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
36
41
  say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
@@ -41,12 +46,22 @@ module SidekiqUniqueJobs
41
46
 
42
47
  no_commands do
43
48
  def console_class
44
- require 'pry'
45
- Pry
46
- rescue LoadError
47
- require 'irb'
49
+ return irb if RUBY_PLATFORM == "JAVA"
50
+
51
+ pry
52
+ end
53
+
54
+ def irb
55
+ require "irb"
48
56
  IRB
49
57
  end
58
+
59
+ def pry
60
+ require "pry"
61
+ Pry
62
+ rescue LoadError, NameError
63
+ irb
64
+ end
50
65
  end
51
66
  end
52
67
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq_unique_jobs/server/middleware'
3
+ require "sidekiq_unique_jobs/server/middleware"
4
4
 
5
5
  module SidekiqUniqueJobs
6
6
  module Client
@@ -38,13 +38,17 @@ module SidekiqUniqueJobs
38
38
  end
39
39
 
40
40
  def locked?
41
- locked = lock.lock
42
- warn_about_duplicate unless locked
43
- locked
41
+ SidekiqUniqueJobs::Job.add_uniqueness(item)
42
+ SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
43
+ locked = lock.lock
44
+ warn_about_duplicate unless locked
45
+ locked
46
+ end
44
47
  end
45
48
 
46
49
  def warn_about_duplicate
47
50
  return unless log_duplicate_payload?
51
+
48
52
  log_warn "payload is not unique #{item}"
49
53
  end
50
54
  end
@@ -1,22 +1,33 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ #
4
+ # Module with constants to avoid string duplication
5
+ #
6
+ # @author Mikael Henriksson <mikael@zoolutions.se>
7
+ #
3
8
  module SidekiqUniqueJobs
4
- ARGS_KEY ||= 'args'
5
- AT_KEY ||= 'at'
6
- CLASS_KEY ||= 'class'
7
- JID_KEY ||= 'jid'
8
- LOCK_EXPIRATION_KEY ||= 'lock_expiration'
9
- LOCK_TIMEOUT_KEY ||= 'lock_timeout'
10
- LOG_DUPLICATE_KEY ||= 'log_duplicate_payload'
11
- QUEUE_KEY ||= 'queue'
12
- UNIQUE_ACROSS_QUEUES_KEY ||= 'unique_across_queues'
13
- UNIQUE_ACROSS_WORKERS_KEY ||= 'unique_across_workers'
14
- UNIQUE_ARGS_KEY ||= 'unique_args'
15
- UNIQUE_DIGEST_KEY ||= 'unique_digest'
16
- UNIQUE_KEY ||= 'unique'
17
- UNIQUE_SET ||= 'unique:keys'
18
- LOCK_KEY ||= 'lock'
19
- ON_CONFLICT_KEY ||= 'on_conflict'
20
- UNIQUE_ON_ALL_QUEUES_KEY ||= 'unique_on_all_queues' # TODO: Remove in v6.1
21
- UNIQUE_PREFIX_KEY ||= 'unique_prefix'
9
+ ARGS_KEY ||= "args"
10
+ APARTMENT ||= "apartment"
11
+ AT_KEY ||= "at"
12
+ CLASS_KEY ||= "class"
13
+ JAVA ||= "java"
14
+ JID_KEY ||= "jid"
15
+ LOCK_DIGEST_KEY ||= "lock_digest"
16
+ LOCK_EXPIRATION_KEY ||= "lock_expiration"
17
+ LOCK_TIMEOUT_KEY ||= "lock_timeout"
18
+ LOCK_TTL_KEY ||= "lock_ttl"
19
+ LOG_DUPLICATE_KEY ||= "log_duplicate_payload"
20
+ QUEUE_KEY ||= "queue"
21
+ UNIQUE_ACROSS_QUEUES_KEY ||= "unique_across_queues"
22
+ UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
23
+ UNIQUE_ARGS_KEY ||= "unique_args"
24
+ UNIQUE_DIGEST_KEY ||= "unique_digest"
25
+ UNIQUE_KEY ||= "unique"
26
+ UNIQUE_SET ||= "unique:keys"
27
+ LOCK_KEY ||= "lock"
28
+ ON_CONFLICT_KEY ||= "on_conflict"
29
+ UNIQUE_ON_ALL_QUEUES_KEY ||= "unique_on_all_queues" # TODO: Remove in v6.1
30
+ UNIQUE_PREFIX_KEY ||= "unique_prefix"
31
+ RETRY_SET ||= "retry"
32
+ SCHEDULE_SET ||= "schedule"
22
33
  end
@@ -6,12 +6,12 @@ module SidekiqUniqueJobs
6
6
  # @author Mikael Henriksson <mikael@zoolutions.se>
7
7
  module Digests
8
8
  DEFAULT_COUNT = 1_000
9
- SCAN_PATTERN = '*'
9
+ SCAN_PATTERN = "*"
10
10
  CHUNK_SIZE = 100
11
11
 
12
12
  include SidekiqUniqueJobs::Logging
13
13
  include SidekiqUniqueJobs::Connection
14
- extend self # rubocop:disable Style/ModuleFunction
14
+ extend self
15
15
 
16
16
  # Return unique digests matching pattern
17
17
  #
@@ -25,8 +25,9 @@ module SidekiqUniqueJobs
25
25
  # Paginate unique digests
26
26
  #
27
27
  # @param [String] pattern a pattern to match with
28
- # @param [Integer] page the current cursor position
29
- # @param [Integer] count the maximum number to match
28
+ # @param [Integer] cursor the maximum number to match
29
+ # @param [Integer] page_size the current cursor position
30
+ #
30
31
  # @return [Array<String>] with unique digests
31
32
  def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
32
33
  redis do |conn|
@@ -54,13 +55,39 @@ module SidekiqUniqueJobs
54
55
  # @raise [ArgumentError] when both pattern and digest are nil
55
56
  # @return [Array<String>] with unique digests
56
57
  def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
58
+ warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
59
+
57
60
  return delete_by_pattern(pattern, count: count) if pattern
58
61
  return delete_by_digest(digest) if digest
59
62
 
60
- raise ArgumentError, 'either digest or pattern need to be provided'
63
+ raise ArgumentError, "either digest or pattern need to be provided"
61
64
  end
62
65
 
63
- private
66
+ # Deletes unique digest either by a digest or pattern
67
+ #
68
+ # @param [String] digest the full digest to delete
69
+ def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
70
+ result, elapsed = timed do
71
+ Scripts.call(:delete_by_digest, nil, keys: [
72
+ UNIQUE_SET,
73
+ digest,
74
+ "#{digest}:EXISTS",
75
+ "#{digest}:GRABBED",
76
+ "#{digest}:AVAILABLE",
77
+ "#{digest}:VERSION",
78
+ "#{digest}:RUN:EXISTS",
79
+ "#{digest}:RUN:GRABBED",
80
+ "#{digest}:RUN:AVAILABLE",
81
+ "#{digest}:RUN:VERSION",
82
+ ])
83
+
84
+ count
85
+ end
86
+
87
+ log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
88
+
89
+ result
90
+ end
64
91
 
65
92
  # Deletes unique digests by pattern
66
93
  #
@@ -79,19 +106,7 @@ module SidekiqUniqueJobs
79
106
  result
80
107
  end
81
108
 
82
- # Get a total count of unique digests
83
- #
84
- # @param [String] digest a key pattern to match with
85
- def delete_by_digest(digest)
86
- result, elapsed = timed do
87
- Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, digest])
88
- count
89
- end
90
-
91
- log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
92
-
93
- result
94
- end
109
+ private
95
110
 
96
111
  def batch_delete(digests) # rubocop:disable Metrics/MethodLength
97
112
  redis do |conn|
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Utility class to append uniqueness to the sidekiq job hash
5
+ #
6
+ # @author Mikael Henriksson <mikael@zoolutions.se>
7
+ module Job
8
+ extend self
9
+
10
+ # Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
11
+ # @return [void] nothing returned here matters
12
+ def add_uniqueness(item)
13
+ add_timeout_and_expiration(item)
14
+ add_unique_args_and_digest(item)
15
+ end
16
+
17
+ private
18
+
19
+ def add_timeout_and_expiration(item)
20
+ calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
21
+ item[LOCK_TIMEOUT_KEY] = calculator.lock_timeout
22
+ item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
23
+ end
24
+
25
+ def add_unique_args_and_digest(item)
26
+ SidekiqUniqueJobs::UniqueArgs.digest(item)
27
+ end
28
+ end
29
+ end