sidekiq-unique-jobs 7.0.0.beta25 → 7.0.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +99 -15
- data/README.md +192 -55
- data/lib/sidekiq-unique-jobs.rb +0 -2
- data/lib/sidekiq_unique_jobs/batch_delete.rb +1 -1
- data/lib/sidekiq_unique_jobs/changelog.rb +2 -2
- data/lib/sidekiq_unique_jobs/cli.rb +1 -1
- data/lib/sidekiq_unique_jobs/config.rb +2 -2
- data/lib/sidekiq_unique_jobs/connection.rb +1 -1
- data/lib/sidekiq_unique_jobs/constants.rb +5 -1
- data/lib/sidekiq_unique_jobs/digests.rb +2 -2
- data/lib/sidekiq_unique_jobs/exceptions.rb +9 -9
- data/lib/sidekiq_unique_jobs/job.rb +2 -2
- data/lib/sidekiq_unique_jobs/json.rb +1 -1
- data/lib/sidekiq_unique_jobs/key.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/client_validator.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/server_validator.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/validator.rb +3 -2
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +2 -2
- data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock_args.rb +5 -5
- data/lib/sidekiq_unique_jobs/lock_config.rb +3 -1
- data/lib/sidekiq_unique_jobs/lock_digest.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock_info.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock_timeout.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock_ttl.rb +1 -1
- data/lib/sidekiq_unique_jobs/locksmith.rb +2 -2
- data/lib/sidekiq_unique_jobs/logging.rb +1 -1
- data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +2 -2
- data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +1 -1
- data/lib/sidekiq_unique_jobs/lua/lock.lua +1 -2
- data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +8 -7
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +9 -2
- data/lib/sidekiq_unique_jobs/middleware.rb +1 -58
- data/lib/sidekiq_unique_jobs/middleware/client.rb +1 -1
- data/lib/sidekiq_unique_jobs/middleware/server.rb +1 -1
- data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/log.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +10 -9
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +2 -2
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +1 -1
- data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +2 -2
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +41 -5
- data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
- data/lib/sidekiq_unique_jobs/orphans/observer.rb +2 -2
- data/lib/sidekiq_unique_jobs/orphans/reaper.rb +15 -1
- data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +10 -3
- data/lib/sidekiq_unique_jobs/redis.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/entity.rb +10 -4
- data/lib/sidekiq_unique_jobs/redis/hash.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/list.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/set.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/string.rb +1 -1
- data/lib/sidekiq_unique_jobs/rspec/matchers.rb +2 -2
- data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +2 -2
- data/lib/sidekiq_unique_jobs/script.rb +1 -1
- data/lib/sidekiq_unique_jobs/script/caller.rb +2 -2
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +5 -5
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +2 -2
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +2 -2
- data/lib/sidekiq_unique_jobs/testing.rb +2 -1
- data/lib/sidekiq_unique_jobs/timing.rb +1 -1
- data/lib/sidekiq_unique_jobs/unlockable.rb +1 -1
- data/lib/sidekiq_unique_jobs/update_version.rb +1 -1
- data/lib/sidekiq_unique_jobs/upgrade_locks.rb +1 -1
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs/version_check.rb +1 -1
- data/lib/sidekiq_unique_jobs/web.rb +8 -9
- data/lib/sidekiq_unique_jobs/web/helpers.rb +1 -1
- metadata +26 -11
- data/lib/sidekiq_unique_jobs/profiler.rb +0 -55
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Orphans
|
5
|
+
#
|
6
|
+
# Class DeleteOrphans provides deletion of orphaned digests
|
7
|
+
#
|
8
|
+
# @note this is a much slower version of the lua script but does not crash redis
|
9
|
+
#
|
10
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
11
|
+
#
|
12
|
+
class NullReaper < Reaper
|
13
|
+
#
|
14
|
+
# Delete orphaned digests
|
15
|
+
#
|
16
|
+
#
|
17
|
+
# @return [Integer] the number of reaped locks
|
18
|
+
#
|
19
|
+
def call
|
20
|
+
# NO OP
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -6,13 +6,13 @@ module SidekiqUniqueJobs
|
|
6
6
|
#
|
7
7
|
# @note this is a much slower version of the lua script but does not crash redis
|
8
8
|
#
|
9
|
-
# @author Mikael Henriksson <mikael@
|
9
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
10
10
|
#
|
11
11
|
module Orphans
|
12
12
|
#
|
13
13
|
# Observes the Orphan::Manager and provides information about each execution
|
14
14
|
#
|
15
|
-
# @author Mikael Henriksson <mikael@
|
15
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
16
16
|
#
|
17
17
|
class Observer
|
18
18
|
include SidekiqUniqueJobs::Logging
|
@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
|
|
7
7
|
#
|
8
8
|
# @note this is a much slower version of the lua script but does not crash redis
|
9
9
|
#
|
10
|
-
# @author Mikael Henriksson <mikael@
|
10
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
11
11
|
#
|
12
12
|
class Reaper
|
13
13
|
include SidekiqUniqueJobs::Connection
|
@@ -17,12 +17,16 @@ module SidekiqUniqueJobs
|
|
17
17
|
|
18
18
|
require_relative "lua_reaper"
|
19
19
|
require_relative "ruby_reaper"
|
20
|
+
require_relative "null_reaper"
|
20
21
|
|
21
22
|
#
|
22
23
|
# @return [Hash<Symbol, SidekiqUniqueJobs::Orphans::Reaper] the current implementation of reapers
|
23
24
|
REAPERS = {
|
24
25
|
lua: SidekiqUniqueJobs::Orphans::LuaReaper,
|
25
26
|
ruby: SidekiqUniqueJobs::Orphans::RubyReaper,
|
27
|
+
none: SidekiqUniqueJobs::Orphans::NullReaper,
|
28
|
+
nil => SidekiqUniqueJobs::Orphans::NullReaper,
|
29
|
+
false => SidekiqUniqueJobs::Orphans::NullReaper,
|
26
30
|
}.freeze
|
27
31
|
|
28
32
|
#
|
@@ -72,6 +76,16 @@ module SidekiqUniqueJobs
|
|
72
76
|
config.reaper
|
73
77
|
end
|
74
78
|
|
79
|
+
#
|
80
|
+
# The configured timeout for the reaper
|
81
|
+
#
|
82
|
+
#
|
83
|
+
# @return [Integer] timeout in seconds
|
84
|
+
#
|
85
|
+
def reaper_timeout
|
86
|
+
config.reaper_timeout
|
87
|
+
end
|
88
|
+
|
75
89
|
#
|
76
90
|
# The number of locks to reap at a time
|
77
91
|
#
|
@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
|
|
7
7
|
#
|
8
8
|
# @note this is a much slower version of the lua script but does not crash redis
|
9
9
|
#
|
10
|
-
# @author Mikael Henriksson <mikael@
|
10
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
11
11
|
#
|
12
12
|
class RubyReaper < Reaper
|
13
13
|
#
|
@@ -117,7 +117,7 @@ module SidekiqUniqueJobs
|
|
117
117
|
end
|
118
118
|
end
|
119
119
|
|
120
|
-
def active?(digest) # rubocop:disable Metrics/MethodLength
|
120
|
+
def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
|
121
121
|
Sidekiq.redis do |conn|
|
122
122
|
procs = conn.sscan_each("processes").to_a
|
123
123
|
return false if procs.empty?
|
@@ -132,7 +132,10 @@ module SidekiqUniqueJobs
|
|
132
132
|
next unless workers.any?
|
133
133
|
|
134
134
|
workers.each_pair do |_tid, job|
|
135
|
-
|
135
|
+
item = load_json(job)
|
136
|
+
|
137
|
+
return true if item.dig(PAYLOAD, LOCK_DIGEST) == digest
|
138
|
+
return true if considered_active?(item[CREATED_AT])
|
136
139
|
end
|
137
140
|
end
|
138
141
|
|
@@ -140,6 +143,10 @@ module SidekiqUniqueJobs
|
|
140
143
|
end
|
141
144
|
end
|
142
145
|
|
146
|
+
def considered_active?(time_f)
|
147
|
+
(Time.now - reaper_timeout).to_f < time_f
|
148
|
+
end
|
149
|
+
|
143
150
|
#
|
144
151
|
# Loops through all the redis queues and yields them one by one
|
145
152
|
#
|
@@ -5,7 +5,7 @@ module SidekiqUniqueJobs
|
|
5
5
|
#
|
6
6
|
# Class Entity functions as a base class for redis types
|
7
7
|
#
|
8
|
-
# @author Mikael Henriksson <mikael@
|
8
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
9
9
|
#
|
10
10
|
class Entity
|
11
11
|
# includes "SidekiqUniqueJobs::Logging"
|
@@ -48,10 +48,10 @@ module SidekiqUniqueJobs
|
|
48
48
|
def exist?
|
49
49
|
redis do |conn|
|
50
50
|
value = conn.exists(key)
|
51
|
-
return true if value.is_a?(TrueClass)
|
52
|
-
return false if value.is_a?(FalseClass)
|
53
51
|
|
54
|
-
value
|
52
|
+
return value if boolean?(value)
|
53
|
+
|
54
|
+
value.to_i.positive?
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
@@ -95,6 +95,12 @@ module SidekiqUniqueJobs
|
|
95
95
|
def count
|
96
96
|
0
|
97
97
|
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def boolean?(value)
|
102
|
+
[TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
|
103
|
+
end
|
98
104
|
end
|
99
105
|
end
|
100
106
|
end
|
@@ -4,13 +4,13 @@ module SidekiqUniqueJobs
|
|
4
4
|
#
|
5
5
|
# We all know what the hell RSpec is no?
|
6
6
|
#
|
7
|
-
# @author Mikael Henriksson <mikael@
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
8
|
#
|
9
9
|
module RSpec
|
10
10
|
#
|
11
11
|
# Module Matchers provides RSpec matcher for your workers
|
12
12
|
#
|
13
|
-
# @author Mikael Henriksson <mikael@
|
13
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
14
14
|
#
|
15
15
|
module Matchers
|
16
16
|
end
|
@@ -5,13 +5,13 @@ module SidekiqUniqueJobs
|
|
5
5
|
#
|
6
6
|
# Module Matchers provides RSpec matcher for your workers
|
7
7
|
#
|
8
|
-
# @author Mikael Henriksson <mikael@
|
8
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
9
9
|
#
|
10
10
|
module Matchers
|
11
11
|
#
|
12
12
|
# Class HaveValidSidekiqOptions validates the unique/lock configuration for a worker.
|
13
13
|
#
|
14
|
-
# @author Mikael Henriksson <mikael@
|
14
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
15
15
|
#
|
16
16
|
class HaveValidSidekiqOptions
|
17
17
|
attr_reader :worker, :lock_config, :sidekiq_options
|
@@ -3,12 +3,12 @@
|
|
3
3
|
module SidekiqUniqueJobs
|
4
4
|
# Interface to dealing with .lua files
|
5
5
|
#
|
6
|
-
# @author Mikael Henriksson <mikael@
|
6
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
7
|
module Script
|
8
8
|
#
|
9
9
|
# Module Caller provides the convenience method #call_script
|
10
10
|
#
|
11
|
-
# @author Mikael Henriksson <mikael@
|
11
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
12
12
|
#
|
13
13
|
module Caller
|
14
14
|
module_function
|
@@ -11,7 +11,7 @@ module Sidekiq
|
|
11
11
|
#
|
12
12
|
# Provides extensions for unlocking jobs that are removed and deleted
|
13
13
|
#
|
14
|
-
# @author Mikael Henriksson <mikael@
|
14
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
15
15
|
#
|
16
16
|
module UniqueExtension
|
17
17
|
#
|
@@ -48,7 +48,7 @@ module Sidekiq
|
|
48
48
|
#
|
49
49
|
# Provides extensions for unlocking jobs that are removed and deleted
|
50
50
|
#
|
51
|
-
# @author Mikael Henriksson <mikael@
|
51
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
52
52
|
#
|
53
53
|
module UniqueExtension
|
54
54
|
#
|
@@ -73,7 +73,7 @@ module Sidekiq
|
|
73
73
|
#
|
74
74
|
# Provides extensions for unlocking jobs that are removed and deleted
|
75
75
|
#
|
76
|
-
# @author Mikael Henriksson <mikael@
|
76
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
77
77
|
#
|
78
78
|
module UniqueExtension
|
79
79
|
#
|
@@ -93,7 +93,7 @@ module Sidekiq
|
|
93
93
|
#
|
94
94
|
# Provides extensions for unlocking jobs that are removed and deleted
|
95
95
|
#
|
96
|
-
# @author Mikael Henriksson <mikael@
|
96
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
97
97
|
#
|
98
98
|
module UniqueExtension
|
99
99
|
#
|
@@ -113,7 +113,7 @@ module Sidekiq
|
|
113
113
|
#
|
114
114
|
# Provides extensions for unlocking jobs that are removed and deleted
|
115
115
|
#
|
116
|
-
# @author Mikael Henriksson <mikael@
|
116
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
117
117
|
#
|
118
118
|
module UniqueExtension
|
119
119
|
#
|
@@ -3,7 +3,7 @@
|
|
3
3
|
#
|
4
4
|
# Contains configuration and utility methods that belongs top level
|
5
5
|
#
|
6
|
-
# @author Mikael Henriksson <mikael@
|
6
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
7
|
module SidekiqUniqueJobs
|
8
8
|
include SidekiqUniqueJobs::Connection
|
9
9
|
extend SidekiqUniqueJobs::JSON
|
@@ -168,7 +168,7 @@ module SidekiqUniqueJobs
|
|
168
168
|
# @option options [Integer] :lock_timeout (default is 0)
|
169
169
|
# @option options [Integer] :lock_ttl (default is 0)
|
170
170
|
# @option options [true,false] :enabled (default is true)
|
171
|
-
# @option options [String] :
|
171
|
+
# @option options [String] :lock_prefix (default is 'uniquejobs')
|
172
172
|
# @option options [Logger] :logger (default is Sidekiq.logger)
|
173
173
|
# @yield control to the caller when given block
|
174
174
|
def configure(options = {})
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module SidekiqUniqueJobs
|
4
4
|
# Module with convenience methods for the Sidekiq::Worker class
|
5
5
|
#
|
6
|
-
# @author Mikael Henriksson <mikael@
|
6
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
7
|
module SidekiqWorkerMethods
|
8
8
|
# Avoids duplicating worker_class.respond_to? in multiple places
|
9
9
|
# @return [true, false]
|
@@ -50,7 +50,7 @@ module SidekiqUniqueJobs
|
|
50
50
|
# @return [Sidekiq::Worker]
|
51
51
|
def worker_class_constantize(klazz = @worker_class)
|
52
52
|
return klazz.class if klazz.is_a?(Sidekiq::Worker) # sidekiq v6.x
|
53
|
-
return klazz
|
53
|
+
return klazz unless klazz.is_a?(String)
|
54
54
|
|
55
55
|
Object.const_get(klazz)
|
56
56
|
rescue NameError => ex
|
@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
|
|
4
4
|
# Utility module to help manage unique keys in redis.
|
5
5
|
# Useful for deleting keys that for whatever reason wasn't deleted
|
6
6
|
#
|
7
|
-
# @author Mikael Henriksson <mikael@
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
8
|
module Unlockable
|
9
9
|
module_function
|
10
10
|
|
@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
|
|
4
4
|
#
|
5
5
|
# Handles checking if a version is compliant with given constraint
|
6
6
|
#
|
7
|
-
# @author Mikael Henriksson <mikael@
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
8
|
#
|
9
9
|
class VersionCheck
|
10
10
|
PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength, Lint/MixedRegexpCaptureTypes
|
@@ -1,18 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
begin
|
4
|
-
require "sidekiq/web"
|
5
|
-
rescue LoadError
|
6
|
-
# client-only usage
|
7
|
-
end
|
8
|
-
|
9
3
|
require_relative "web/helpers"
|
10
4
|
|
11
5
|
module SidekiqUniqueJobs
|
12
6
|
# Utility module to help manage unique keys in redis.
|
13
7
|
# Useful for deleting keys that for whatever reason wasn't deleted
|
14
8
|
#
|
15
|
-
# @author Mikael Henriksson <mikael@
|
9
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
16
10
|
module Web
|
17
11
|
def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
18
12
|
app.helpers do
|
@@ -59,8 +53,13 @@ module SidekiqUniqueJobs
|
|
59
53
|
end
|
60
54
|
end
|
61
55
|
|
62
|
-
|
63
|
-
|
56
|
+
begin
|
57
|
+
require "delegate" unless defined?(DelegateClass)
|
58
|
+
require "sidekiq/web" unless defined?(Sidekiq::Web)
|
59
|
+
|
60
|
+
Sidekiq::Web.register(SidekiqUniqueJobs::Web)
|
64
61
|
Sidekiq::Web.tabs["Locks"] = "locks"
|
65
62
|
Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
|
63
|
+
rescue NameError, LoadError => ex
|
64
|
+
SidekiqUniqueJobs.logger.error(ex)
|
66
65
|
end
|