sidekiq-unique-jobs 6.0.6 → 6.0.25
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +768 -150
- data/README.md +123 -77
- data/bin/uniquejobs +2 -2
- data/lib/sidekiq-unique-jobs.rb +1 -1
- data/lib/sidekiq_unique_jobs/cli.rb +27 -12
- data/lib/sidekiq_unique_jobs/client/middleware.rb +8 -4
- data/lib/sidekiq_unique_jobs/constants.rb +29 -18
- data/lib/sidekiq_unique_jobs/digests.rb +34 -19
- data/lib/sidekiq_unique_jobs/job.rb +29 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +14 -18
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +20 -4
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +7 -3
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +1 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +9 -3
- data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +0 -8
- data/lib/sidekiq_unique_jobs/locksmith.rb +59 -32
- data/lib/sidekiq_unique_jobs/logging.rb +14 -0
- data/lib/sidekiq_unique_jobs/middleware.rb +26 -9
- data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +3 -3
- data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +7 -2
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict.rb +12 -7
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +8 -8
- data/lib/sidekiq_unique_jobs/scripts.rb +38 -9
- data/lib/sidekiq_unique_jobs/server/middleware.rb +18 -6
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +1 -1
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +88 -0
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +13 -4
- data/lib/sidekiq_unique_jobs/testing.rb +3 -3
- data/lib/sidekiq_unique_jobs/timeout/calculator.rb +2 -1
- data/lib/sidekiq_unique_jobs/timeout.rb +1 -1
- data/lib/sidekiq_unique_jobs/unique_args.rb +7 -4
- data/lib/sidekiq_unique_jobs/util.rb +8 -4
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs/version_check.rb +95 -0
- data/lib/sidekiq_unique_jobs/web/helpers.rb +7 -6
- data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +4 -0
- data/lib/sidekiq_unique_jobs/web.rb +19 -12
- data/lib/sidekiq_unique_jobs.rb +33 -107
- data/lib/tasks/changelog.rake +23 -0
- data/redis/convert_legacy_lock.lua +13 -0
- data/redis/delete_by_digest.lua +10 -11
- data/redis/delete_job_by_digest.lua +6 -4
- data/redis/lock.lua +14 -10
- data/redis/unlock.lua +13 -11
- metadata +88 -108
- data/.codeclimate.yml +0 -35
- data/.csslintrc +0 -2
- data/.dockerignore +0 -4
- data/.editorconfig +0 -14
- data/.eslintignore +0 -1
- data/.eslintrc +0 -213
- data/.fasterer.yml +0 -23
- data/.github/ISSUE_TEMPLATE/bug_report.md +0 -31
- data/.github/ISSUE_TEMPLATE/feature_request.md +0 -17
- data/.gitignore +0 -26
- data/.reek.yml +0 -87
- data/.rspec +0 -2
- data/.rubocop.yml +0 -127
- data/.simplecov +0 -20
- data/.travis.yml +0 -49
- data/.yardopts +0 -7
- data/Appraisals +0 -25
- data/CODE_OF_CONDUCT.md +0 -74
- data/Gemfile +0 -24
- data/Guardfile +0 -55
- data/Rakefile +0 -12
- data/_config.yml +0 -1
- data/assets/unique_digests_1.png +0 -0
- data/assets/unique_digests_2.png +0 -0
- data/bin/bench +0 -20
- data/examples/another_unique_job.rb +0 -15
- data/examples/custom_queue_job.rb +0 -12
- data/examples/custom_queue_job_with_filter_method.rb +0 -13
- data/examples/custom_queue_job_with_filter_proc.rb +0 -16
- data/examples/expiring_job.rb +0 -12
- data/examples/inline_worker.rb +0 -12
- data/examples/just_a_worker.rb +0 -13
- data/examples/long_running_job.rb +0 -14
- data/examples/main_job.rb +0 -14
- data/examples/my_job.rb +0 -12
- data/examples/my_unique_job.rb +0 -15
- data/examples/my_unique_job_with_filter_method.rb +0 -21
- data/examples/my_unique_job_with_filter_proc.rb +0 -19
- data/examples/notify_worker.rb +0 -14
- data/examples/plain_class.rb +0 -13
- data/examples/simple_worker.rb +0 -15
- data/examples/spawn_simple_worker.rb +0 -12
- data/examples/test_class.rb +0 -9
- data/examples/unique_across_workers_job.rb +0 -20
- data/examples/unique_job_on_conflict_raise.rb +0 -14
- data/examples/unique_job_on_conflict_reject.rb +0 -14
- data/examples/unique_job_on_conflict_reschedule.rb +0 -14
- data/examples/unique_job_with_conditional_parameter.rb +0 -18
- data/examples/unique_job_with_filter_method.rb +0 -21
- data/examples/unique_job_with_nil_unique_args.rb +0 -20
- data/examples/unique_job_with_no_unique_args_method.rb +0 -16
- data/examples/unique_job_withthout_unique_args_parameter.rb +0 -18
- data/examples/unique_on_all_queues_job.rb +0 -16
- data/examples/until_and_while_executing_job.rb +0 -17
- data/examples/until_executed_2_job.rb +0 -24
- data/examples/until_executed_job.rb +0 -25
- data/examples/until_executing_job.rb +0 -11
- data/examples/until_expired_job.rb +0 -12
- data/examples/until_global_expired_job.rb +0 -12
- data/examples/while_executing_job.rb +0 -15
- data/examples/while_executing_reject_job.rb +0 -14
- data/examples/without_argument_job.rb +0 -13
- data/sidekiq-unique-jobs.gemspec +0 -42
data/README.md
CHANGED
@@ -1,43 +1,49 @@
|
|
1
|
-
# SidekiqUniqueJobs [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
1
|
+
# SidekiqUniqueJobs [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.com/mhenrixon/sidekiq-unique-jobs.svg?branch=v6.x)](https://travis-ci.com/mhenrixon/sidekiq-unique-jobs) [![Code Climate](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs.png)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs) [![Test Coverage](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/badges/coverage.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/coverage)
|
2
|
+
|
3
|
+
<!-- MarkdownTOC -->
|
4
|
+
|
5
|
+
- [Introduction](#introduction)
|
6
|
+
- [Documentation](#documentation)
|
7
|
+
- [Requirements](#requirements)
|
8
|
+
- [ActiveJob](#activejob)
|
9
|
+
- [redis-namespace](#redis-namespace)
|
10
|
+
- [Installation](#installation)
|
11
|
+
- [Support Me](#support-me)
|
12
|
+
- [General Information](#general-information)
|
13
|
+
- [Options](#options)
|
14
|
+
- [Lock Expiration](#lock-expiration)
|
15
|
+
- [Lock Timeout](#lock-timeout)
|
16
|
+
- [Unique Across Queues](#unique-across-queues)
|
17
|
+
- [Unique Across Workers](#unique-across-workers)
|
18
|
+
- [Locks](#locks)
|
19
|
+
- [Until Executing](#until-executing)
|
20
|
+
- [Until Executed](#until-executed)
|
21
|
+
- [Until Timeout](#until-timeout)
|
22
|
+
- [Unique Until And While Executing](#unique-until-and-while-executing)
|
23
|
+
- [While Executing](#while-executing)
|
24
|
+
- [Conflict Strategy](#conflict-strategy)
|
25
|
+
- [Log](#log)
|
26
|
+
- [Raise](#raise)
|
27
|
+
- [Reject](#reject)
|
28
|
+
- [Replace](#replace)
|
29
|
+
- [Reschedule](#reschedule)
|
30
|
+
- [Usage](#usage)
|
31
|
+
- [Finer Control over Uniqueness](#finer-control-over-uniqueness)
|
32
|
+
- [After Unlock Callback](#after-unlock-callback)
|
33
|
+
- [Logging](#logging)
|
34
|
+
- [Cleanup Dead Locks](#cleanup-dead-locks)
|
35
|
+
- [Other Sidekiq gems](#other-sidekiq-gems)
|
36
|
+
- [sidekiq-global_id](#sidekiq-global_id)
|
37
|
+
- [Debugging](#debugging)
|
38
|
+
- [Sidekiq Web](#sidekiq-web)
|
39
|
+
- [Show Unique Digests](#show-unique-digests)
|
40
|
+
- [Show keys for digest](#show-keys-for-digest)
|
41
|
+
- [Communication](#communication)
|
42
|
+
- [Testing](#testing)
|
43
|
+
- [Contributing](#contributing)
|
44
|
+
- [Contributors](#contributors)
|
45
|
+
|
46
|
+
<!-- /MarkdownTOC -->
|
41
47
|
|
42
48
|
## Introduction
|
43
49
|
|
@@ -45,37 +51,50 @@ The goal of this gem is to ensure your Sidekiq jobs are unique. We do this by cr
|
|
45
51
|
|
46
52
|
## Documentation
|
47
53
|
|
48
|
-
This is the documentation for the master branch. You can find the documentation for each release by navigating to its tag:
|
54
|
+
This is the documentation for the master branch. You can find the documentation for each release by navigating to its tag: [v5.0.10][]
|
49
55
|
|
50
56
|
Below are links to the latest major versions (4 & 5):
|
51
57
|
|
52
|
-
- [v5.0.10]
|
53
|
-
- [v4.0.18]
|
58
|
+
- [v5.0.10][]
|
59
|
+
- [v4.0.18][]
|
54
60
|
|
55
61
|
## Requirements
|
56
62
|
|
57
|
-
See
|
63
|
+
See [Sidekiq requirements][] for what is required. Starting from 5.0.0 only sidekiq >= 4 and MRI >= 2.2. ActiveJob is not supported
|
58
64
|
|
59
|
-
|
65
|
+
### ActiveJob
|
66
|
+
|
67
|
+
Version 6 requires Redis >= 3 and pure Sidekiq, no ActiveJob supported anymore. See [About ActiveJob](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/About-ActiveJob) for why. It simply is too complex and generates more issues than I can handle given how little timer I have to spend on this project.
|
68
|
+
|
69
|
+
### redis-namespace
|
70
|
+
|
71
|
+
Will not be officially supported anymore. Since Mike [won't support redis-namespace](https://github.com/mperham/sidekiq/issues/3366#issuecomment-284270120) neither will I.
|
72
|
+
|
73
|
+
[Read this](http://www.mikeperham.com/2017/04/10/migrating-from-redis-namespace/) for how to migrate away from namespacing.
|
60
74
|
|
61
75
|
## Installation
|
62
76
|
|
63
77
|
Add this line to your application's Gemfile:
|
64
78
|
|
65
|
-
|
79
|
+
```
|
80
|
+
gem 'sidekiq-unique-jobs'
|
81
|
+
```
|
66
82
|
|
67
83
|
And then execute:
|
68
84
|
|
69
|
-
|
85
|
+
```
|
86
|
+
bundle
|
87
|
+
```
|
70
88
|
|
71
89
|
Or install it yourself as:
|
72
90
|
|
73
|
-
|
74
|
-
|
91
|
+
```
|
92
|
+
gem install sidekiq-unique-jobs
|
93
|
+
```
|
75
94
|
|
76
95
|
## Support Me
|
77
96
|
|
78
|
-
Want to show me some ❤️ for the hard work I do on this gem? You can use the following PayPal link
|
97
|
+
Want to show me some ❤️ for the hard work I do on this gem? You can use the following [PayPal link][]. Any amount is welcome and let me tell you it feels good to be appreciated. Even a dollar makes me super excited about all of this.
|
79
98
|
|
80
99
|
## General Information
|
81
100
|
|
@@ -87,15 +106,13 @@ See [Locking & Unlocking](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/
|
|
87
106
|
|
88
107
|
### Lock Expiration
|
89
108
|
|
90
|
-
|
91
|
-
|
92
|
-
Since the client and the server are disconnected and not running inside the same process, setting a lock expiration is probably not what you want. Any keys that are used by this gem WILL be removed at the time of the expiration. For jobs that are scheduled in the future the key will expire when that job is scheduled + whatever expiration you have set.
|
109
|
+
Lock expiration is used for two things. For the `UntilExpired` job releases the lock upon expiry. This is done from the client.
|
93
110
|
|
94
|
-
|
111
|
+
Since v6.0.11 the other locks will expire after the server is done processing.
|
95
112
|
|
96
113
|
```ruby
|
97
114
|
sidekiq_options lock_expiration: nil # default - don't expire keys
|
98
|
-
sidekiq_options lock_expiration: 20.days # expire this lock in 20 days
|
115
|
+
sidekiq_options lock_expiration: 20.days.to_i # expire this lock in 20 days
|
99
116
|
```
|
100
117
|
|
101
118
|
### Lock Timeout
|
@@ -115,7 +132,7 @@ This configuration option is slightly misleading. It doesn't disregard the queue
|
|
115
132
|
```ruby
|
116
133
|
class Worker
|
117
134
|
include Sidekiq::Worker
|
118
|
-
|
135
|
+
|
119
136
|
sidekiq_options unique_across_queues: true, queue: 'default'
|
120
137
|
|
121
138
|
def perform(args); end
|
@@ -131,7 +148,7 @@ This configuration option is slightly misleading. It doesn't disregard the worke
|
|
131
148
|
```ruby
|
132
149
|
class WorkerOne
|
133
150
|
include Sidekiq::Worker
|
134
|
-
|
151
|
+
|
135
152
|
sidekiq_options unique_across_workers: true, queue: 'default'
|
136
153
|
|
137
154
|
def perform(args); end
|
@@ -139,17 +156,17 @@ end
|
|
139
156
|
|
140
157
|
class WorkerTwo
|
141
158
|
include Sidekiq::Worker
|
142
|
-
|
159
|
+
|
143
160
|
sidekiq_options unique_across_workers: true, queue: 'default'
|
144
161
|
|
145
162
|
def perform(args); end
|
146
163
|
end
|
147
164
|
|
148
165
|
|
149
|
-
WorkerOne.perform_async(1)
|
166
|
+
WorkerOne.perform_async(1)
|
150
167
|
# => 'the jobs unique id'
|
151
168
|
|
152
|
-
WorkerTwo.perform_async(1)
|
169
|
+
WorkerTwo.perform_async(1)
|
153
170
|
# => nil because WorkerOne just stole the lock
|
154
171
|
```
|
155
172
|
|
@@ -220,7 +237,7 @@ In the console you should see something like:
|
|
220
237
|
|
221
238
|
## Conflict Strategy
|
222
239
|
|
223
|
-
Decides how we handle conflict. We can either reject the job to the dead queue or reschedule it. Both are useful for jobs that absolutely need to run and have been configured to use the lock `WhileExecuting` that is used only by the sidekiq server process.
|
240
|
+
Decides how we handle conflict. We can either reject the job to the dead queue or reschedule it. Both are useful for jobs that absolutely need to run and have been configured to use the lock `WhileExecuting` that is used only by the sidekiq server process.
|
224
241
|
|
225
242
|
The last one is log which can be be used with the lock `UntilExecuted` and `UntilExpired`. Now we write a log entry saying the job could not be pushed because it is a duplicate of another job with the same arguments
|
226
243
|
|
@@ -246,9 +263,9 @@ This strategy is intended to be used with `WhileExecuting` and will push the job
|
|
246
263
|
|
247
264
|
This strategy is intended to be used with client locks like `UntilExecuted`.
|
248
265
|
It will delete any existing job for these arguments from retry, schedule and
|
249
|
-
queue and retry the lock again.
|
266
|
+
queue and retry the lock again.
|
250
267
|
|
251
|
-
This is slightly dangerous and should probably only be used for jobs that are
|
268
|
+
This is slightly dangerous and should probably only be used for jobs that are
|
252
269
|
always scheduled in the future. Currently only attempting to retry one time.
|
253
270
|
|
254
271
|
`sidekiq_options lock: :until_executed, on_conflict: :replace`
|
@@ -322,16 +339,22 @@ end
|
|
322
339
|
|
323
340
|
### After Unlock Callback
|
324
341
|
|
325
|
-
If you need to perform any additional work after the lock has been released you can provide an `#after_unlock` instance method. The method will be called when the lock has been unlocked. Most times this means after yield but there are two exceptions to that.
|
342
|
+
If you need to perform any additional work after the lock has been released you can provide an `#after_unlock` instance method. The method will be called when the lock has been unlocked. Most times this means after yield but there are two exceptions to that.
|
326
343
|
|
327
344
|
**Exception 1:** UntilExecuting unlocks and calls back before yielding.
|
328
345
|
**Exception 2:** UntilExpired expires eventually, no after_unlock hook is called.
|
329
346
|
|
347
|
+
**NOTE:** _It is also possible to write this code as a class method._
|
348
|
+
|
330
349
|
```ruby
|
331
350
|
class UniqueJobWithFilterMethod
|
332
351
|
include Sidekiq::Worker
|
333
352
|
sidekiq_options lock: :while_executing,
|
334
353
|
|
354
|
+
def self.after_unlock
|
355
|
+
# block has yielded and lock is released
|
356
|
+
end
|
357
|
+
|
335
358
|
def after_unlock
|
336
359
|
# block has yielded and lock is released
|
337
360
|
end
|
@@ -356,12 +379,12 @@ end
|
|
356
379
|
|
357
380
|
### Cleanup Dead Locks
|
358
381
|
|
359
|
-
For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class.
|
382
|
+
For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class. This is deprecated in Sidekiq 6.0
|
360
383
|
|
361
384
|
```ruby
|
362
385
|
class MyWorker
|
363
386
|
sidekiq_retries_exhausted do |msg, _ex|
|
364
|
-
SidekiqUniqueJobs::Digests.
|
387
|
+
SidekiqUniqueJobs::Digests.delete_by_digest(msg['unique_digest']) if msg['unique_digest']
|
365
388
|
end
|
366
389
|
end
|
367
390
|
```
|
@@ -372,11 +395,29 @@ Starting in v5.1, Sidekiq can also fire a global callback when a job dies:
|
|
372
395
|
# this goes in your initializer
|
373
396
|
Sidekiq.configure_server do |config|
|
374
397
|
config.death_handlers << ->(job, _ex) do
|
375
|
-
SidekiqUniqueJobs::Digests.
|
398
|
+
SidekiqUniqueJobs::Digests.delete_by_digest(job['unique_digest']) if job['unique_digest']
|
376
399
|
end
|
377
400
|
end
|
378
401
|
```
|
379
402
|
|
403
|
+
### Other Sidekiq gems
|
404
|
+
|
405
|
+
#### sidekiq-global_id
|
406
|
+
|
407
|
+
It was reported in [#235](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/235) that the order of the Sidekiq middleware needs to be as follows.
|
408
|
+
|
409
|
+
```ruby
|
410
|
+
Sidekiq.client_middleware do |chain|
|
411
|
+
chain.add Sidekiq::GlobalId::ClientMiddleware
|
412
|
+
chain.add SidekiqUniqueJobs::Client::Middleware
|
413
|
+
end
|
414
|
+
|
415
|
+
Sidekiq.server_middleware do |chain|
|
416
|
+
chain.add SidekiqUniqueJobs::Server::Middleware
|
417
|
+
chain.add Sidekiq::GlobalId::ServerMiddleware
|
418
|
+
end
|
419
|
+
```
|
420
|
+
|
380
421
|
## Debugging
|
381
422
|
|
382
423
|
There are several ways of removing keys that are stuck. The prefered way is by using the unique extension to `Sidekiq::Web`. The old console and command line versions still work but might be deprecated in the future. It is better to search for the digest itself and delete the keys matching that digest.
|
@@ -391,12 +432,11 @@ require 'sidekiq_unique_jobs/web'
|
|
391
432
|
mount Sidekiq::Web, at: '/sidekiq'
|
392
433
|
```
|
393
434
|
|
394
|
-
There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web`
|
435
|
+
There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web`
|
395
436
|
already does this.
|
396
437
|
|
397
438
|
To filter/search for keys we can use the wildcard `*`. If we have a unique digest `'uniquejobs:9e9b5ce5d423d3ea470977004b50ff84` we can search for it by enter `*ff84` and it should return all digests that end with `ff84`.
|
398
439
|
|
399
|
-
|
400
440
|
#### Show Unique Digests
|
401
441
|
|
402
442
|
![Unique Digests](assets/unique_digests_1.png)
|
@@ -413,7 +453,7 @@ There is a [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](
|
|
413
453
|
|
414
454
|
This has been probably the most confusing part of this gem. People get really confused with how unreliable the unique jobs have been. I there for decided to do what Mike is doing for sidekiq enterprise. Read the section about unique jobs.
|
415
455
|
|
416
|
-
|
456
|
+
[Enterprise unique jobs][]
|
417
457
|
|
418
458
|
```ruby
|
419
459
|
SidekiqUniqueJobs.configure do |config|
|
@@ -443,7 +483,7 @@ RSpec.describe Workers::CoolOne do
|
|
443
483
|
|
444
484
|
it 'prevents duplicate jobs from being scheduled' do
|
445
485
|
SidekiqUniqueJobs.use_config(enabled: true) do
|
446
|
-
expect(described_class.
|
486
|
+
expect(described_class.perform_in(3600, 1)).not_to eq(nil)
|
447
487
|
expect(described_class.perform_async(1)).to eq(nil)
|
448
488
|
end
|
449
489
|
end
|
@@ -462,12 +502,18 @@ I would strongly suggest you let this gem test uniqueness. If you care about how
|
|
462
502
|
## Contributing
|
463
503
|
|
464
504
|
1. Fork it
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
505
|
+
1. Create your feature branch (`git checkout -b my-new-feature`)
|
506
|
+
1. Commit your changes (`git commit -am 'Add some feature'`)
|
507
|
+
1. Push to the branch (`git push origin my-new-feature`)
|
508
|
+
1. Create new Pull Request
|
469
509
|
|
510
|
+
## Contributors
|
470
511
|
|
471
|
-
|
512
|
+
You can find a list of contributors over on [Contributors][]
|
472
513
|
|
473
|
-
|
514
|
+
[v5.0.10]: https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v5.0.10.
|
515
|
+
[v4.0.18]: https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v4.0.18
|
516
|
+
[Sidekiq requirements]: https://github.com/mperham/sidekiq#requirements
|
517
|
+
[Enterprise unique jobs]: https://www.dailydrip.com/topics/sidekiq/drips/sidekiq-enterprise-unique-jobs
|
518
|
+
[Contributors]: https://github.com/mhenrixon/sidekiq-unique-jobs/graphs/contributors
|
519
|
+
[Paypal link https://paypal.me/mhenrixon]: https://paypal.me/mhenrixon
|
data/bin/uniquejobs
CHANGED
data/lib/sidekiq-unique-jobs.rb
CHANGED
@@ -1,24 +1,29 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "thor"
|
4
4
|
|
5
5
|
module SidekiqUniqueJobs
|
6
|
+
#
|
7
|
+
# Command line interface for unique jobs
|
8
|
+
#
|
9
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
10
|
+
#
|
6
11
|
class Cli < Thor
|
7
12
|
def self.banner(command, _namespace = nil, _subcommand = false)
|
8
13
|
"jobs #{@package_name} #{command.usage}"
|
9
14
|
end
|
10
15
|
|
11
|
-
desc
|
12
|
-
option :count, aliases: :c, type: :numeric, default: 1000, desc:
|
13
|
-
def keys(pattern =
|
16
|
+
desc "keys PATTERN", "list all unique keys and their expiry time"
|
17
|
+
option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
|
18
|
+
def keys(pattern = "*")
|
14
19
|
keys = Util.keys(pattern, options[:count])
|
15
20
|
say "Found #{keys.size} keys matching '#{pattern}':"
|
16
21
|
print_in_columns(keys.sort) if keys.any?
|
17
22
|
end
|
18
23
|
|
19
|
-
desc
|
20
|
-
option :dry_run, aliases: :d, type: :boolean, desc:
|
21
|
-
option :count, aliases: :c, type: :numeric, default: 1000, desc:
|
24
|
+
desc "del PATTERN", "deletes unique keys from redis by pattern"
|
25
|
+
option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
|
26
|
+
option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
|
22
27
|
def del(pattern)
|
23
28
|
max_count = options[:count]
|
24
29
|
if options[:dry_run]
|
@@ -30,7 +35,7 @@ module SidekiqUniqueJobs
|
|
30
35
|
end
|
31
36
|
end
|
32
37
|
|
33
|
-
desc
|
38
|
+
desc "console", "drop into a console with easy access to helper methods"
|
34
39
|
def console
|
35
40
|
say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
|
36
41
|
say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
|
@@ -41,12 +46,22 @@ module SidekiqUniqueJobs
|
|
41
46
|
|
42
47
|
no_commands do
|
43
48
|
def console_class
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
49
|
+
return irb if RUBY_PLATFORM == "JAVA"
|
50
|
+
|
51
|
+
pry
|
52
|
+
end
|
53
|
+
|
54
|
+
def irb
|
55
|
+
require "irb"
|
48
56
|
IRB
|
49
57
|
end
|
58
|
+
|
59
|
+
def pry
|
60
|
+
require "pry"
|
61
|
+
Pry
|
62
|
+
rescue LoadError, NameError
|
63
|
+
irb
|
64
|
+
end
|
50
65
|
end
|
51
66
|
end
|
52
67
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "sidekiq_unique_jobs/server/middleware"
|
4
4
|
|
5
5
|
module SidekiqUniqueJobs
|
6
6
|
module Client
|
@@ -38,13 +38,17 @@ module SidekiqUniqueJobs
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def locked?
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
SidekiqUniqueJobs::Job.add_uniqueness(item)
|
42
|
+
SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
|
43
|
+
locked = lock.lock
|
44
|
+
warn_about_duplicate unless locked
|
45
|
+
locked
|
46
|
+
end
|
44
47
|
end
|
45
48
|
|
46
49
|
def warn_about_duplicate
|
47
50
|
return unless log_duplicate_payload?
|
51
|
+
|
48
52
|
log_warn "payload is not unique #{item}"
|
49
53
|
end
|
50
54
|
end
|
@@ -1,22 +1,33 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
#
|
4
|
+
# Module with constants to avoid string duplication
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
|
+
#
|
3
8
|
module SidekiqUniqueJobs
|
4
|
-
ARGS_KEY ||=
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
9
|
+
ARGS_KEY ||= "args"
|
10
|
+
APARTMENT ||= "apartment"
|
11
|
+
AT_KEY ||= "at"
|
12
|
+
CLASS_KEY ||= "class"
|
13
|
+
JAVA ||= "java"
|
14
|
+
JID_KEY ||= "jid"
|
15
|
+
LOCK_DIGEST_KEY ||= "lock_digest"
|
16
|
+
LOCK_EXPIRATION_KEY ||= "lock_expiration"
|
17
|
+
LOCK_TIMEOUT_KEY ||= "lock_timeout"
|
18
|
+
LOCK_TTL_KEY ||= "lock_ttl"
|
19
|
+
LOG_DUPLICATE_KEY ||= "log_duplicate_payload"
|
20
|
+
QUEUE_KEY ||= "queue"
|
21
|
+
UNIQUE_ACROSS_QUEUES_KEY ||= "unique_across_queues"
|
22
|
+
UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
|
23
|
+
UNIQUE_ARGS_KEY ||= "unique_args"
|
24
|
+
UNIQUE_DIGEST_KEY ||= "unique_digest"
|
25
|
+
UNIQUE_KEY ||= "unique"
|
26
|
+
UNIQUE_SET ||= "unique:keys"
|
27
|
+
LOCK_KEY ||= "lock"
|
28
|
+
ON_CONFLICT_KEY ||= "on_conflict"
|
29
|
+
UNIQUE_ON_ALL_QUEUES_KEY ||= "unique_on_all_queues" # TODO: Remove in v6.1
|
30
|
+
UNIQUE_PREFIX_KEY ||= "unique_prefix"
|
31
|
+
RETRY_SET ||= "retry"
|
32
|
+
SCHEDULE_SET ||= "schedule"
|
22
33
|
end
|
@@ -6,12 +6,12 @@ module SidekiqUniqueJobs
|
|
6
6
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
7
|
module Digests
|
8
8
|
DEFAULT_COUNT = 1_000
|
9
|
-
SCAN_PATTERN =
|
9
|
+
SCAN_PATTERN = "*"
|
10
10
|
CHUNK_SIZE = 100
|
11
11
|
|
12
12
|
include SidekiqUniqueJobs::Logging
|
13
13
|
include SidekiqUniqueJobs::Connection
|
14
|
-
extend self
|
14
|
+
extend self
|
15
15
|
|
16
16
|
# Return unique digests matching pattern
|
17
17
|
#
|
@@ -25,8 +25,9 @@ module SidekiqUniqueJobs
|
|
25
25
|
# Paginate unique digests
|
26
26
|
#
|
27
27
|
# @param [String] pattern a pattern to match with
|
28
|
-
# @param [Integer]
|
29
|
-
# @param [Integer]
|
28
|
+
# @param [Integer] cursor the maximum number to match
|
29
|
+
# @param [Integer] page_size the current cursor position
|
30
|
+
#
|
30
31
|
# @return [Array<String>] with unique digests
|
31
32
|
def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
|
32
33
|
redis do |conn|
|
@@ -54,13 +55,39 @@ module SidekiqUniqueJobs
|
|
54
55
|
# @raise [ArgumentError] when both pattern and digest are nil
|
55
56
|
# @return [Array<String>] with unique digests
|
56
57
|
def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
|
58
|
+
warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
|
59
|
+
|
57
60
|
return delete_by_pattern(pattern, count: count) if pattern
|
58
61
|
return delete_by_digest(digest) if digest
|
59
62
|
|
60
|
-
raise ArgumentError,
|
63
|
+
raise ArgumentError, "either digest or pattern need to be provided"
|
61
64
|
end
|
62
65
|
|
63
|
-
|
66
|
+
# Deletes unique digest either by a digest or pattern
|
67
|
+
#
|
68
|
+
# @param [String] digest the full digest to delete
|
69
|
+
def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
|
70
|
+
result, elapsed = timed do
|
71
|
+
Scripts.call(:delete_by_digest, nil, keys: [
|
72
|
+
UNIQUE_SET,
|
73
|
+
digest,
|
74
|
+
"#{digest}:EXISTS",
|
75
|
+
"#{digest}:GRABBED",
|
76
|
+
"#{digest}:AVAILABLE",
|
77
|
+
"#{digest}:VERSION",
|
78
|
+
"#{digest}:RUN:EXISTS",
|
79
|
+
"#{digest}:RUN:GRABBED",
|
80
|
+
"#{digest}:RUN:AVAILABLE",
|
81
|
+
"#{digest}:RUN:VERSION",
|
82
|
+
])
|
83
|
+
|
84
|
+
count
|
85
|
+
end
|
86
|
+
|
87
|
+
log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
|
88
|
+
|
89
|
+
result
|
90
|
+
end
|
64
91
|
|
65
92
|
# Deletes unique digests by pattern
|
66
93
|
#
|
@@ -79,19 +106,7 @@ module SidekiqUniqueJobs
|
|
79
106
|
result
|
80
107
|
end
|
81
108
|
|
82
|
-
|
83
|
-
#
|
84
|
-
# @param [String] digest a key pattern to match with
|
85
|
-
def delete_by_digest(digest)
|
86
|
-
result, elapsed = timed do
|
87
|
-
Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, digest])
|
88
|
-
count
|
89
|
-
end
|
90
|
-
|
91
|
-
log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
|
92
|
-
|
93
|
-
result
|
94
|
-
end
|
109
|
+
private
|
95
110
|
|
96
111
|
def batch_delete(digests) # rubocop:disable Metrics/MethodLength
|
97
112
|
redis do |conn|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
# Utility class to append uniqueness to the sidekiq job hash
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
|
+
module Job
|
8
|
+
extend self
|
9
|
+
|
10
|
+
# Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
|
11
|
+
# @return [void] nothing returned here matters
|
12
|
+
def add_uniqueness(item)
|
13
|
+
add_timeout_and_expiration(item)
|
14
|
+
add_unique_args_and_digest(item)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def add_timeout_and_expiration(item)
|
20
|
+
calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
|
21
|
+
item[LOCK_TIMEOUT_KEY] = calculator.lock_timeout
|
22
|
+
item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
|
23
|
+
end
|
24
|
+
|
25
|
+
def add_unique_args_and_digest(item)
|
26
|
+
SidekiqUniqueJobs::UniqueArgs.digest(item)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|