sidekiq-unique-jobs 6.0.8 → 6.0.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.codeclimate.yml +10 -13
- data/.editorconfig +1 -1
- data/.gitignore +11 -22
- data/.mdlrc +1 -0
- data/.reek.yml +1 -0
- data/.rspec +1 -1
- data/.rubocop.yml +58 -30
- data/.travis.yml +14 -7
- data/.yardopts +1 -7
- data/CHANGELOG.md +704 -160
- data/Gemfile +19 -17
- data/Guardfile +0 -29
- data/README.md +77 -65
- data/Rakefile +22 -5
- data/bin/bench +5 -5
- data/bin/uniquejobs +2 -2
- data/examples/custom_queue_job.rb +1 -1
- data/examples/custom_queue_job_with_filter_method.rb +1 -1
- data/examples/custom_queue_job_with_filter_proc.rb +2 -2
- data/examples/my_unique_job_with_filter_method.rb +1 -1
- data/examples/my_unique_job_with_filter_proc.rb +1 -1
- data/examples/unique_job_with_filter_method.rb +1 -1
- data/lib/sidekiq-unique-jobs.rb +1 -1
- data/lib/sidekiq_unique_jobs.rb +32 -107
- data/lib/sidekiq_unique_jobs/cli.rb +15 -10
- data/lib/sidekiq_unique_jobs/client/middleware.rb +1 -1
- data/lib/sidekiq_unique_jobs/constants.rb +25 -20
- data/lib/sidekiq_unique_jobs/digests.rb +5 -4
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +3 -3
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +1 -1
- data/lib/sidekiq_unique_jobs/locksmith.rb +4 -4
- data/lib/sidekiq_unique_jobs/logging.rb +1 -1
- data/lib/sidekiq_unique_jobs/middleware.rb +9 -4
- data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict.rb +12 -7
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +3 -3
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +1 -1
- data/lib/sidekiq_unique_jobs/scripts.rb +5 -5
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +1 -1
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +75 -0
- data/lib/sidekiq_unique_jobs/testing.rb +3 -3
- data/lib/sidekiq_unique_jobs/timeout.rb +1 -1
- data/lib/sidekiq_unique_jobs/unique_args.rb +2 -2
- data/lib/sidekiq_unique_jobs/util.rb +3 -3
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs/web.rb +13 -8
- data/lib/sidekiq_unique_jobs/web/helpers.rb +2 -2
- data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +4 -0
- data/lib/tasks/changelog.rake +23 -0
- data/sidekiq-unique-jobs.gemspec +48 -27
- data/update_docs.sh +37 -0
- metadata +68 -33
- data/.csslintrc +0 -2
- data/.dockerignore +0 -4
- data/.eslintignore +0 -1
- data/.eslintrc +0 -213
data/lib/sidekiq-unique-jobs.rb
CHANGED
data/lib/sidekiq_unique_jobs.rb
CHANGED
@@ -1,109 +1,34 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
|
6
|
-
require
|
7
|
-
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
13
|
-
require
|
14
|
-
require
|
15
|
-
require
|
16
|
-
require
|
17
|
-
require
|
18
|
-
require
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
require
|
25
|
-
require
|
26
|
-
require
|
27
|
-
require
|
28
|
-
require
|
29
|
-
require
|
30
|
-
require
|
31
|
-
require
|
32
|
-
require
|
33
|
-
|
34
|
-
require
|
35
|
-
|
36
|
-
# Namespace for this gem
|
37
|
-
#
|
38
|
-
# Contains configuration and utility methods that belongs top level
|
39
|
-
#
|
40
|
-
# @author Mikael Henriksson <mikael@zoolutions.se>
|
41
|
-
module SidekiqUniqueJobs
|
42
|
-
include SidekiqUniqueJobs::Connection
|
43
|
-
|
44
|
-
module_function
|
45
|
-
|
46
|
-
Config = Concurrent::MutableStruct.new(
|
47
|
-
:default_lock_timeout,
|
48
|
-
:enabled,
|
49
|
-
:unique_prefix,
|
50
|
-
:logger,
|
51
|
-
)
|
52
|
-
|
53
|
-
# The current configuration (See: {.configure} on how to configure)
|
54
|
-
def config
|
55
|
-
# Arguments here need to match the definition of the new class (see above)
|
56
|
-
@config ||= Config.new(
|
57
|
-
0,
|
58
|
-
true,
|
59
|
-
'uniquejobs',
|
60
|
-
Sidekiq.logger,
|
61
|
-
)
|
62
|
-
end
|
63
|
-
|
64
|
-
# The current logger
|
65
|
-
# @return [Logger] the configured logger
|
66
|
-
def logger
|
67
|
-
config.logger
|
68
|
-
end
|
69
|
-
|
70
|
-
# Set a new logger
|
71
|
-
# @param [Logger] other a new logger
|
72
|
-
def logger=(other)
|
73
|
-
config.logger = other
|
74
|
-
end
|
75
|
-
|
76
|
-
# Change global configuration while yielding
|
77
|
-
# @yield control to the caller
|
78
|
-
def use_config(tmp_config)
|
79
|
-
fail ::ArgumentError, "#{name}.#{__method__} needs a block" unless block_given?
|
80
|
-
|
81
|
-
old_config = config.to_h
|
82
|
-
configure(tmp_config)
|
83
|
-
yield
|
84
|
-
configure(old_config)
|
85
|
-
end
|
86
|
-
|
87
|
-
# Configure the gem
|
88
|
-
#
|
89
|
-
# This is usually called once at startup of an application
|
90
|
-
# @param [Hash] options global gem options
|
91
|
-
# @option options [Integer] :default_lock_timeout (default is 0)
|
92
|
-
# @option options [true,false] :enabled (default is true)
|
93
|
-
# @option options [String] :unique_prefix (default is 'uniquejobs')
|
94
|
-
# @option options [Logger] :logger (default is Sidekiq.logger)
|
95
|
-
# @yield control to the caller when given block
|
96
|
-
def configure(options = {})
|
97
|
-
if block_given?
|
98
|
-
yield config
|
99
|
-
else
|
100
|
-
options.each do |key, val|
|
101
|
-
config.send("#{key}=", val)
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
def redis_version
|
107
|
-
@redis_version ||= redis { |conn| conn.info('server')['redis_version'] }
|
108
|
-
end
|
109
|
-
end
|
3
|
+
require "forwardable"
|
4
|
+
require "concurrent/mutable_struct"
|
5
|
+
|
6
|
+
require "sidekiq_unique_jobs/version"
|
7
|
+
require "sidekiq_unique_jobs/constants"
|
8
|
+
require "sidekiq_unique_jobs/logging"
|
9
|
+
require "sidekiq_unique_jobs/sidekiq_worker_methods"
|
10
|
+
require "sidekiq_unique_jobs/connection"
|
11
|
+
require "sidekiq_unique_jobs/exceptions"
|
12
|
+
require "sidekiq_unique_jobs/job"
|
13
|
+
require "sidekiq_unique_jobs/util"
|
14
|
+
require "sidekiq_unique_jobs/digests"
|
15
|
+
require "sidekiq_unique_jobs/cli"
|
16
|
+
require "sidekiq_unique_jobs/core_ext"
|
17
|
+
require "sidekiq_unique_jobs/timeout"
|
18
|
+
require "sidekiq_unique_jobs/scripts"
|
19
|
+
require "sidekiq_unique_jobs/unique_args"
|
20
|
+
require "sidekiq_unique_jobs/unlockable"
|
21
|
+
require "sidekiq_unique_jobs/locksmith"
|
22
|
+
require "sidekiq_unique_jobs/lock/base_lock"
|
23
|
+
require "sidekiq_unique_jobs/lock/until_executed"
|
24
|
+
require "sidekiq_unique_jobs/lock/until_executing"
|
25
|
+
require "sidekiq_unique_jobs/lock/until_expired"
|
26
|
+
require "sidekiq_unique_jobs/lock/while_executing"
|
27
|
+
require "sidekiq_unique_jobs/lock/while_executing_reject"
|
28
|
+
require "sidekiq_unique_jobs/lock/until_and_while_executing"
|
29
|
+
require "sidekiq_unique_jobs/options_with_fallback"
|
30
|
+
require "sidekiq_unique_jobs/middleware"
|
31
|
+
require "sidekiq_unique_jobs/sidekiq_unique_ext"
|
32
|
+
require "sidekiq_unique_jobs/on_conflict"
|
33
|
+
|
34
|
+
require "sidekiq_unique_jobs/sidekiq_unique_jobs"
|
@@ -1,24 +1,29 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "thor"
|
4
4
|
|
5
5
|
module SidekiqUniqueJobs
|
6
|
+
#
|
7
|
+
# Command line interface for unique jobs
|
8
|
+
#
|
9
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
10
|
+
#
|
6
11
|
class Cli < Thor
|
7
12
|
def self.banner(command, _namespace = nil, _subcommand = false)
|
8
13
|
"jobs #{@package_name} #{command.usage}"
|
9
14
|
end
|
10
15
|
|
11
|
-
desc
|
12
|
-
option :count, aliases: :c, type: :numeric, default: 1000, desc:
|
13
|
-
def keys(pattern =
|
16
|
+
desc "keys PATTERN", "list all unique keys and their expiry time"
|
17
|
+
option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
|
18
|
+
def keys(pattern = "*")
|
14
19
|
keys = Util.keys(pattern, options[:count])
|
15
20
|
say "Found #{keys.size} keys matching '#{pattern}':"
|
16
21
|
print_in_columns(keys.sort) if keys.any?
|
17
22
|
end
|
18
23
|
|
19
|
-
desc
|
20
|
-
option :dry_run, aliases: :d, type: :boolean, desc:
|
21
|
-
option :count, aliases: :c, type: :numeric, default: 1000, desc:
|
24
|
+
desc "del PATTERN", "deletes unique keys from redis by pattern"
|
25
|
+
option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
|
26
|
+
option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
|
22
27
|
def del(pattern)
|
23
28
|
max_count = options[:count]
|
24
29
|
if options[:dry_run]
|
@@ -30,7 +35,7 @@ module SidekiqUniqueJobs
|
|
30
35
|
end
|
31
36
|
end
|
32
37
|
|
33
|
-
desc
|
38
|
+
desc "console", "drop into a console with easy access to helper methods"
|
34
39
|
def console
|
35
40
|
say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
|
36
41
|
say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
|
@@ -41,10 +46,10 @@ module SidekiqUniqueJobs
|
|
41
46
|
|
42
47
|
no_commands do
|
43
48
|
def console_class
|
44
|
-
require
|
49
|
+
require "pry"
|
45
50
|
Pry
|
46
51
|
rescue LoadError
|
47
|
-
require
|
52
|
+
require "irb"
|
48
53
|
IRB
|
49
54
|
end
|
50
55
|
end
|
@@ -1,24 +1,29 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
#
|
4
|
+
# Module with constants to avoid string duplication
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
|
+
#
|
3
8
|
module SidekiqUniqueJobs
|
4
|
-
ARGS_KEY ||=
|
5
|
-
AT_KEY ||=
|
6
|
-
CLASS_KEY ||=
|
7
|
-
JID_KEY ||=
|
8
|
-
LOCK_EXPIRATION_KEY ||=
|
9
|
-
LOCK_TIMEOUT_KEY ||=
|
10
|
-
LOG_DUPLICATE_KEY ||=
|
11
|
-
QUEUE_KEY ||=
|
12
|
-
UNIQUE_ACROSS_QUEUES_KEY ||=
|
13
|
-
UNIQUE_ACROSS_WORKERS_KEY ||=
|
14
|
-
UNIQUE_ARGS_KEY ||=
|
15
|
-
UNIQUE_DIGEST_KEY ||=
|
16
|
-
UNIQUE_KEY ||=
|
17
|
-
UNIQUE_SET ||=
|
18
|
-
LOCK_KEY ||=
|
19
|
-
ON_CONFLICT_KEY ||=
|
20
|
-
UNIQUE_ON_ALL_QUEUES_KEY ||=
|
21
|
-
UNIQUE_PREFIX_KEY ||=
|
22
|
-
RETRY_SET ||=
|
23
|
-
SCHEDULE_SET ||=
|
9
|
+
ARGS_KEY ||= "args"
|
10
|
+
AT_KEY ||= "at"
|
11
|
+
CLASS_KEY ||= "class"
|
12
|
+
JID_KEY ||= "jid"
|
13
|
+
LOCK_EXPIRATION_KEY ||= "lock_expiration"
|
14
|
+
LOCK_TIMEOUT_KEY ||= "lock_timeout"
|
15
|
+
LOG_DUPLICATE_KEY ||= "log_duplicate_payload"
|
16
|
+
QUEUE_KEY ||= "queue"
|
17
|
+
UNIQUE_ACROSS_QUEUES_KEY ||= "unique_across_queues"
|
18
|
+
UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
|
19
|
+
UNIQUE_ARGS_KEY ||= "unique_args"
|
20
|
+
UNIQUE_DIGEST_KEY ||= "unique_digest"
|
21
|
+
UNIQUE_KEY ||= "unique"
|
22
|
+
UNIQUE_SET ||= "unique:keys"
|
23
|
+
LOCK_KEY ||= "lock"
|
24
|
+
ON_CONFLICT_KEY ||= "on_conflict"
|
25
|
+
UNIQUE_ON_ALL_QUEUES_KEY ||= "unique_on_all_queues" # TODO: Remove in v6.1
|
26
|
+
UNIQUE_PREFIX_KEY ||= "unique_prefix"
|
27
|
+
RETRY_SET ||= "retry"
|
28
|
+
SCHEDULE_SET ||= "schedule"
|
24
29
|
end
|
@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
|
|
6
6
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
7
|
module Digests
|
8
8
|
DEFAULT_COUNT = 1_000
|
9
|
-
SCAN_PATTERN =
|
9
|
+
SCAN_PATTERN = "*"
|
10
10
|
CHUNK_SIZE = 100
|
11
11
|
|
12
12
|
include SidekiqUniqueJobs::Logging
|
@@ -25,8 +25,9 @@ module SidekiqUniqueJobs
|
|
25
25
|
# Paginate unique digests
|
26
26
|
#
|
27
27
|
# @param [String] pattern a pattern to match with
|
28
|
-
# @param [Integer]
|
29
|
-
# @param [Integer]
|
28
|
+
# @param [Integer] cursor the maximum number to match
|
29
|
+
# @param [Integer] page_size the current cursor position
|
30
|
+
#
|
30
31
|
# @return [Array<String>] with unique digests
|
31
32
|
def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
|
32
33
|
redis do |conn|
|
@@ -57,7 +58,7 @@ module SidekiqUniqueJobs
|
|
57
58
|
return delete_by_pattern(pattern, count: count) if pattern
|
58
59
|
return delete_by_digest(digest) if digest
|
59
60
|
|
60
|
-
raise ArgumentError,
|
61
|
+
raise ArgumentError, "either digest or pattern need to be provided"
|
61
62
|
end
|
62
63
|
|
63
64
|
private
|
@@ -108,14 +108,14 @@ module SidekiqUniqueJobs
|
|
108
108
|
def with_cleanup
|
109
109
|
yield
|
110
110
|
rescue Sidekiq::Shutdown
|
111
|
-
log_info(
|
111
|
+
log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
|
112
112
|
raise
|
113
113
|
else
|
114
114
|
unlock_with_callback
|
115
115
|
end
|
116
116
|
|
117
117
|
def unlock_with_callback
|
118
|
-
return log_warn(
|
118
|
+
return log_warn("might need to be unlocked manually") unless unlock
|
119
119
|
|
120
120
|
callback_safely
|
121
121
|
item[JID_KEY]
|
@@ -124,7 +124,7 @@ module SidekiqUniqueJobs
|
|
124
124
|
def callback_safely
|
125
125
|
callback&.call
|
126
126
|
rescue StandardError
|
127
|
-
log_warn(
|
127
|
+
log_warn("unlocked successfully but the #after_unlock callback failed!")
|
128
128
|
raise
|
129
129
|
end
|
130
130
|
|
@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
|
|
11
11
|
#
|
12
12
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
13
13
|
class WhileExecuting < BaseLock
|
14
|
-
RUN_SUFFIX ||=
|
14
|
+
RUN_SUFFIX ||= ":RUN"
|
15
15
|
|
16
16
|
# @param [Hash] item the Sidekiq job hash
|
17
17
|
# @param [Proc] callback callback to call after unlock
|
@@ -133,19 +133,19 @@ module SidekiqUniqueJobs
|
|
133
133
|
end
|
134
134
|
|
135
135
|
def available_key
|
136
|
-
@available_key ||= namespaced_key(
|
136
|
+
@available_key ||= namespaced_key("AVAILABLE")
|
137
137
|
end
|
138
138
|
|
139
139
|
def exists_key
|
140
|
-
@exists_key ||= namespaced_key(
|
140
|
+
@exists_key ||= namespaced_key("EXISTS")
|
141
141
|
end
|
142
142
|
|
143
143
|
def grabbed_key
|
144
|
-
@grabbed_key ||= namespaced_key(
|
144
|
+
@grabbed_key ||= namespaced_key("GRABBED")
|
145
145
|
end
|
146
146
|
|
147
147
|
def version_key
|
148
|
-
@version_key ||= namespaced_key(
|
148
|
+
@version_key ||= namespaced_key("VERSION")
|
149
149
|
end
|
150
150
|
|
151
151
|
def namespaced_key(variable)
|
@@ -1,8 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "sidekiq"
|
4
4
|
|
5
5
|
module SidekiqUniqueJobs
|
6
|
+
#
|
7
|
+
# Provides the sidekiq middleware that makes the gem work
|
8
|
+
#
|
9
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
10
|
+
#
|
6
11
|
module Middleware
|
7
12
|
def self.extended(base)
|
8
13
|
base.class_eval do
|
@@ -18,12 +23,12 @@ module SidekiqUniqueJobs
|
|
18
23
|
def configure_server_middleware
|
19
24
|
Sidekiq.configure_server do |config|
|
20
25
|
config.client_middleware do |chain|
|
21
|
-
require
|
26
|
+
require "sidekiq_unique_jobs/client/middleware"
|
22
27
|
chain.add SidekiqUniqueJobs::Client::Middleware
|
23
28
|
end
|
24
29
|
|
25
30
|
config.server_middleware do |chain|
|
26
|
-
require
|
31
|
+
require "sidekiq_unique_jobs/server/middleware"
|
27
32
|
chain.add SidekiqUniqueJobs::Server::Middleware
|
28
33
|
end
|
29
34
|
end
|
@@ -32,7 +37,7 @@ module SidekiqUniqueJobs
|
|
32
37
|
def configure_client_middleware
|
33
38
|
Sidekiq.configure_client do |config|
|
34
39
|
config.client_middleware do |chain|
|
35
|
-
require
|
40
|
+
require "sidekiq_unique_jobs/client/middleware"
|
36
41
|
chain.add SidekiqUniqueJobs::Client::Middleware
|
37
42
|
end
|
38
43
|
end
|
@@ -1,14 +1,19 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
require_relative
|
6
|
-
require_relative
|
7
|
-
require_relative
|
8
|
-
require_relative
|
9
|
-
require_relative
|
3
|
+
require_relative "on_conflict/strategy"
|
4
|
+
require_relative "on_conflict/null_strategy"
|
5
|
+
require_relative "on_conflict/log"
|
6
|
+
require_relative "on_conflict/raise"
|
7
|
+
require_relative "on_conflict/reject"
|
8
|
+
require_relative "on_conflict/replace"
|
9
|
+
require_relative "on_conflict/reschedule"
|
10
10
|
|
11
11
|
module SidekiqUniqueJobs
|
12
|
+
#
|
13
|
+
# Provides lock conflict resolutions
|
14
|
+
#
|
15
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
16
|
+
#
|
12
17
|
module OnConflict
|
13
18
|
STRATEGIES = {
|
14
19
|
log: OnConflict::Log,
|