sidekiq-unique-jobs 6.0.16 → 6.0.23
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/README.md +22 -2
- data/lib/sidekiq_unique_jobs/cli.rb +11 -3
- data/lib/sidekiq_unique_jobs/constants.rb +2 -0
- data/lib/sidekiq_unique_jobs/digests.rb +28 -14
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +2 -3
- data/lib/sidekiq_unique_jobs/locksmith.rb +3 -3
- data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +1 -1
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +1 -1
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +4 -3
- data/lib/sidekiq_unique_jobs/unique_args.rb +2 -1
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs/version_check.rb +1 -1
- data/lib/sidekiq_unique_jobs/web.rb +5 -3
- data/redis/delete_by_digest.lua +10 -11
- data/redis/lock.lua +2 -1
- metadata +15 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6d6ae6134868752af2c74e22877099bf7d6d93113f149ba35881111007cf2249
|
4
|
+
data.tar.gz: e59c474b8658e73fc85636c88114d848f52b49068e76447abfef0712fe357434
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c4b7b71b5ddfff0fe559c70ec0067c4f6d2d6269632f9c9a01903a40506f385e4898cfb9b57c9d88634a1af98a9353ce9eaa74f1aa62e7e5619428984b52e99c
|
7
|
+
data.tar.gz: c187ac55219ce9cccb812250c29e8509d1ea3e400218ddce3f6148065ec75a509f7f1a056d63bc0f712936e98361b261b96a216099c750464de298ac2896d59d
|
data/README.md
CHANGED
@@ -32,6 +32,8 @@
|
|
32
32
|
- [After Unlock Callback](#after-unlock-callback)
|
33
33
|
- [Logging](#logging)
|
34
34
|
- [Cleanup Dead Locks](#cleanup-dead-locks)
|
35
|
+
- [Other Sidekiq gems](#other-sidekiq-gems)
|
36
|
+
- [sidekiq-global_id](#sidekiq-global_id)
|
35
37
|
- [Debugging](#debugging)
|
36
38
|
- [Sidekiq Web](#sidekiq-web)
|
37
39
|
- [Show Unique Digests](#show-unique-digests)
|
@@ -376,7 +378,7 @@ For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required
|
|
376
378
|
```ruby
|
377
379
|
class MyWorker
|
378
380
|
sidekiq_retries_exhausted do |msg, _ex|
|
379
|
-
SidekiqUniqueJobs::Digests.
|
381
|
+
SidekiqUniqueJobs::Digests.delete_by_digest(msg['unique_digest']) if msg['unique_digest']
|
380
382
|
end
|
381
383
|
end
|
382
384
|
```
|
@@ -387,11 +389,29 @@ Starting in v5.1, Sidekiq can also fire a global callback when a job dies:
|
|
387
389
|
# this goes in your initializer
|
388
390
|
Sidekiq.configure_server do |config|
|
389
391
|
config.death_handlers << ->(job, _ex) do
|
390
|
-
SidekiqUniqueJobs::Digests.
|
392
|
+
SidekiqUniqueJobs::Digests.delete_by_digest(job['unique_digest']) if job['unique_digest']
|
391
393
|
end
|
392
394
|
end
|
393
395
|
```
|
394
396
|
|
397
|
+
### Other Sidekiq gems
|
398
|
+
|
399
|
+
#### sidekiq-global_id
|
400
|
+
|
401
|
+
It was reported in [#235](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/235) that the order of the Sidekiq middleware needs to be as follows.
|
402
|
+
|
403
|
+
```ruby
|
404
|
+
Sidekiq.client_middleware do |chain|
|
405
|
+
chain.add Sidekiq::GlobalId::ClientMiddleware
|
406
|
+
chain.add SidekiqUniqueJobs::Client::Middleware
|
407
|
+
end
|
408
|
+
|
409
|
+
Sidekiq.server_middleware do |chain|
|
410
|
+
chain.add SidekiqUniqueJobs::Server::Middleware
|
411
|
+
chain.add Sidekiq::GlobalId::ServerMiddleware
|
412
|
+
end
|
413
|
+
```
|
414
|
+
|
395
415
|
## Debugging
|
396
416
|
|
397
417
|
There are several ways of removing keys that are stuck. The prefered way is by using the unique extension to `Sidekiq::Web`. The old console and command line versions still work but might be deprecated in the future. It is better to search for the digest itself and delete the keys matching that digest.
|
@@ -46,13 +46,21 @@ module SidekiqUniqueJobs
|
|
46
46
|
|
47
47
|
no_commands do
|
48
48
|
def console_class
|
49
|
-
return
|
49
|
+
return irb if RUBY_PLATFORM == "JAVA"
|
50
50
|
|
51
|
+
pry
|
52
|
+
end
|
53
|
+
|
54
|
+
def irb
|
55
|
+
require "irb"
|
56
|
+
IRB
|
57
|
+
end
|
58
|
+
|
59
|
+
def pry
|
51
60
|
require "pry"
|
52
61
|
Pry
|
53
62
|
rescue LoadError, NameError
|
54
|
-
|
55
|
-
IRB
|
63
|
+
irb
|
56
64
|
end
|
57
65
|
end
|
58
66
|
end
|
@@ -11,8 +11,10 @@ module SidekiqUniqueJobs
|
|
11
11
|
CLASS_KEY ||= "class"
|
12
12
|
JAVA ||= "java"
|
13
13
|
JID_KEY ||= "jid"
|
14
|
+
LOCK_DIGEST_KEY ||= "lock_digest"
|
14
15
|
LOCK_EXPIRATION_KEY ||= "lock_expiration"
|
15
16
|
LOCK_TIMEOUT_KEY ||= "lock_timeout"
|
17
|
+
LOCK_TTL_KEY ||= "lock_ttl"
|
16
18
|
LOG_DUPLICATE_KEY ||= "log_duplicate_payload"
|
17
19
|
QUEUE_KEY ||= "queue"
|
18
20
|
UNIQUE_ACROSS_QUEUES_KEY ||= "unique_across_queues"
|
@@ -55,13 +55,39 @@ module SidekiqUniqueJobs
|
|
55
55
|
# @raise [ArgumentError] when both pattern and digest are nil
|
56
56
|
# @return [Array<String>] with unique digests
|
57
57
|
def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
|
58
|
+
warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
|
59
|
+
|
58
60
|
return delete_by_pattern(pattern, count: count) if pattern
|
59
61
|
return delete_by_digest(digest) if digest
|
60
62
|
|
61
63
|
raise ArgumentError, "either digest or pattern need to be provided"
|
62
64
|
end
|
63
65
|
|
64
|
-
|
66
|
+
# Deletes unique digest either by a digest or pattern
|
67
|
+
#
|
68
|
+
# @param [String] digest the full digest to delete
|
69
|
+
def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
|
70
|
+
result, elapsed = timed do
|
71
|
+
Scripts.call(:delete_by_digest, nil, keys: [
|
72
|
+
UNIQUE_SET,
|
73
|
+
digest,
|
74
|
+
"#{digest}:EXISTS",
|
75
|
+
"#{digest}:GRABBED",
|
76
|
+
"#{digest}:AVAILABLE",
|
77
|
+
"#{digest}:VERSION",
|
78
|
+
"#{digest}:RUN:EXISTS",
|
79
|
+
"#{digest}:RUN:GRABBED",
|
80
|
+
"#{digest}:RUN:AVAILABLE",
|
81
|
+
"#{digest}:RUN:VERSION",
|
82
|
+
])
|
83
|
+
|
84
|
+
count
|
85
|
+
end
|
86
|
+
|
87
|
+
log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
|
88
|
+
|
89
|
+
result
|
90
|
+
end
|
65
91
|
|
66
92
|
# Deletes unique digests by pattern
|
67
93
|
#
|
@@ -80,19 +106,7 @@ module SidekiqUniqueJobs
|
|
80
106
|
result
|
81
107
|
end
|
82
108
|
|
83
|
-
|
84
|
-
#
|
85
|
-
# @param [String] digest a key pattern to match with
|
86
|
-
def delete_by_digest(digest)
|
87
|
-
result, elapsed = timed do
|
88
|
-
Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, digest])
|
89
|
-
count
|
90
|
-
end
|
91
|
-
|
92
|
-
log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
|
93
|
-
|
94
|
-
result
|
95
|
-
end
|
109
|
+
private
|
96
110
|
|
97
111
|
def batch_delete(digests) # rubocop:disable Metrics/MethodLength
|
98
112
|
redis do |conn|
|
@@ -118,11 +118,11 @@ module SidekiqUniqueJobs
|
|
118
118
|
return log_warn("might need to be unlocked manually") unless unlock
|
119
119
|
|
120
120
|
callback_safely
|
121
|
-
item[JID_KEY]
|
122
121
|
end
|
123
122
|
|
124
123
|
def callback_safely
|
125
124
|
callback&.call
|
125
|
+
item[JID_KEY]
|
126
126
|
rescue StandardError
|
127
127
|
log_warn("unlocked successfully but the #after_unlock callback failed!")
|
128
128
|
raise
|
@@ -33,14 +33,13 @@ module SidekiqUniqueJobs
|
|
33
33
|
# These jobs are locked in the server process not from the client
|
34
34
|
# @yield to the worker class perform method
|
35
35
|
def execute
|
36
|
-
return strategy
|
36
|
+
return strategy&.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
|
37
37
|
|
38
38
|
yield
|
39
|
+
unlock_with_callback
|
39
40
|
rescue Exception # rubocop:disable Lint/RescueException
|
40
41
|
delete!
|
41
42
|
raise
|
42
|
-
else
|
43
|
-
unlock_with_callback
|
44
43
|
end
|
45
44
|
|
46
45
|
private
|
@@ -15,10 +15,10 @@ module SidekiqUniqueJobs
|
|
15
15
|
# @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
|
16
16
|
def initialize(item, redis_pool = nil)
|
17
17
|
# @concurrency = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
|
18
|
-
@ttl = item[LOCK_EXPIRATION_KEY]
|
18
|
+
@ttl = item[LOCK_EXPIRATION_KEY] || item[LOCK_TTL_KEY]
|
19
19
|
@jid = item[JID_KEY]
|
20
|
-
@unique_digest = item[UNIQUE_DIGEST_KEY]
|
21
|
-
@lock_type = item[LOCK_KEY]
|
20
|
+
@unique_digest = item[UNIQUE_DIGEST_KEY] || item[LOCK_DIGEST_KEY]
|
21
|
+
@lock_type = item[LOCK_KEY] || item[UNIQUE_KEY]
|
22
22
|
@lock_type &&= @lock_type.to_sym
|
23
23
|
@redis_pool = redis_pool
|
24
24
|
end
|
@@ -29,7 +29,7 @@ module SidekiqUniqueJobs
|
|
29
29
|
# The Sidekiq::Worker implementation
|
30
30
|
# @return [Sidekiq::Worker]
|
31
31
|
def worker_class
|
32
|
-
@_worker_class ||= worker_class_constantize # rubocop:disable Naming/MemoizedInstanceVariableName
|
32
|
+
@_worker_class ||= worker_class_constantize(@worker_class) # rubocop:disable Naming/MemoizedInstanceVariableName
|
33
33
|
end
|
34
34
|
|
35
35
|
# The hook to call after a successful unlock
|
@@ -42,8 +42,9 @@ module SidekiqUniqueJobs
|
|
42
42
|
# failing back to the original argument when the constant can't be found
|
43
43
|
#
|
44
44
|
# @return [Sidekiq::Worker]
|
45
|
-
def worker_class_constantize(klazz
|
46
|
-
return klazz
|
45
|
+
def worker_class_constantize(klazz)
|
46
|
+
return klazz.class if klazz.is_a?(Sidekiq::Worker) # sidekiq v6.x
|
47
|
+
return klazz unless klazz.is_a?(String)
|
47
48
|
|
48
49
|
Object.const_get(klazz)
|
49
50
|
rescue NameError => ex
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "digest"
|
4
|
+
require "openssl"
|
4
5
|
require "sidekiq_unique_jobs/normalizer"
|
5
6
|
|
6
7
|
module SidekiqUniqueJobs
|
@@ -47,7 +48,7 @@ module SidekiqUniqueJobs
|
|
47
48
|
# Creates a namespaced unique digest based on the {#digestable_hash} and the {#unique_prefix}
|
48
49
|
# @return [String] a unique digest
|
49
50
|
def create_digest
|
50
|
-
digest = Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
|
51
|
+
digest = OpenSSL::Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
|
51
52
|
"#{unique_prefix}:#{digest}"
|
52
53
|
end
|
53
54
|
|
@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
|
|
7
7
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
8
8
|
#
|
9
9
|
class VersionCheck
|
10
|
-
PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable
|
10
|
+
PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength
|
11
11
|
|
12
12
|
#
|
13
13
|
# Checks if a version is consrtaint is satisfied
|
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
+
require "delegate"
|
5
|
+
require "rack"
|
4
6
|
require "sidekiq/web"
|
5
|
-
rescue LoadError # rubocop:disable Lint/
|
7
|
+
rescue LoadError # rubocop:disable Lint/SuppressedException
|
6
8
|
# client-only usage
|
7
9
|
end
|
8
10
|
|
@@ -32,7 +34,7 @@ module SidekiqUniqueJobs
|
|
32
34
|
end
|
33
35
|
|
34
36
|
app.get "/unique_digests/delete_all" do
|
35
|
-
SidekiqUniqueJobs::Digests.
|
37
|
+
SidekiqUniqueJobs::Digests.delete_by_pattern("*", count: SidekiqUniqueJobs::Digests.count)
|
36
38
|
redirect_to :unique_digests
|
37
39
|
end
|
38
40
|
|
@@ -44,7 +46,7 @@ module SidekiqUniqueJobs
|
|
44
46
|
end
|
45
47
|
|
46
48
|
app.get "/unique_digests/:digest/delete" do
|
47
|
-
SidekiqUniqueJobs::Digests.
|
49
|
+
SidekiqUniqueJobs::Digests.delete_by_digest(params[:digest])
|
48
50
|
redirect_to :unique_digests
|
49
51
|
end
|
50
52
|
end
|
data/redis/delete_by_digest.lua
CHANGED
@@ -1,15 +1,14 @@
|
|
1
1
|
-- redis.replicate_commands();
|
2
|
-
local unique_keys
|
3
|
-
local unique_digest
|
4
|
-
|
5
|
-
local
|
6
|
-
local
|
7
|
-
local
|
8
|
-
local
|
9
|
-
local
|
10
|
-
local
|
11
|
-
local
|
12
|
-
local run_version_key = unique_digest .. ':RUN:VERSION'
|
2
|
+
local unique_keys = KEYS[1]
|
3
|
+
local unique_digest = KEYS[2]
|
4
|
+
local exists_key = KEYS[3]
|
5
|
+
local grabbed_key = KEYS[4]
|
6
|
+
local available_key = KEYS[5]
|
7
|
+
local version_key = KEYS[6]
|
8
|
+
local run_exists_key = KEYS[7]
|
9
|
+
local run_grabbed_key = KEYS[8]
|
10
|
+
local run_available_key = KEYS[9]
|
11
|
+
local run_version_key = KEYS[10]
|
13
12
|
|
14
13
|
local count = redis.call('SREM', unique_keys, unique_digest)
|
15
14
|
redis.call('DEL', exists_key)
|
data/redis/lock.lua
CHANGED
@@ -49,7 +49,8 @@ redis.call('RPUSH', available_key, job_id)
|
|
49
49
|
|
50
50
|
-- The client should only set ttl for until_expired
|
51
51
|
-- The server should set ttl for all other jobs
|
52
|
-
|
52
|
+
-- redis.log(redis.LOG_WARNING, "lock: " .. lock .. ", ttl: " .. tostring(ttl))
|
53
|
+
if lock == "until_expired" and ttl and ttl > 0 then
|
53
54
|
-- We can't keep the key here because it will otherwise never be deleted
|
54
55
|
redis.call('SREM', unique_keys, unique_digest)
|
55
56
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-unique-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.0.
|
4
|
+
version: 6.0.23
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mikael Henriksson
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-09-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -54,16 +54,22 @@ dependencies:
|
|
54
54
|
name: thor
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|
57
|
-
- - "
|
57
|
+
- - ">="
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
version: '0'
|
59
|
+
version: '0.20'
|
60
|
+
- - "<"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '2.0'
|
60
63
|
type: :runtime
|
61
64
|
prerelease: false
|
62
65
|
version_requirements: !ruby/object:Gem::Requirement
|
63
66
|
requirements:
|
64
|
-
- - "
|
67
|
+
- - ">="
|
65
68
|
- !ruby/object:Gem::Version
|
66
|
-
version: '0'
|
69
|
+
version: '0.20'
|
70
|
+
- - "<"
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '2.0'
|
67
73
|
- !ruby/object:Gem::Dependency
|
68
74
|
name: bundler
|
69
75
|
requirement: !ruby/object:Gem::Requirement
|
@@ -296,13 +302,7 @@ metadata:
|
|
296
302
|
documentation_uri: https://mhenrixon.github.io/sidekiq-unique-jobs
|
297
303
|
source_code_uri: https://github.com/mhenrixon/sidekiq-unique-jobs
|
298
304
|
changelog_uri: https://github.com/mhenrixon/sidekiq-unique-jobs/CHANGELOG.md
|
299
|
-
post_install_message:
|
300
|
-
Please either install v6.0.19 or change your `unique: :lock_type` to `lock: :lock_type`
|
301
|
-
This version will not unlock otherwise.
|
302
|
-
|
303
|
-
See: https://github.com/mhenrixon/sidekiq-unique-jobs/issues/471
|
304
|
-
And: https://github.com/mhenrixon/sidekiq-unique-jobs/pull/435
|
305
|
-
And: https://github.com/mhenrixon/sidekiq-unique-jobs/issues/471
|
305
|
+
post_install_message:
|
306
306
|
rdoc_options: []
|
307
307
|
require_paths:
|
308
308
|
- lib
|
@@ -318,7 +318,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
318
318
|
version: '0'
|
319
319
|
requirements: []
|
320
320
|
rubygems_version: 3.1.2
|
321
|
-
signing_key:
|
321
|
+
signing_key:
|
322
322
|
specification_version: 4
|
323
323
|
summary: Sidekiq middleware that prevents duplicates jobs
|
324
324
|
test_files: []
|