sidekiq 6.0.0 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +383 -2
- data/LICENSE +3 -3
- data/README.md +13 -10
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +74 -66
- data/bin/sidekiqmon +5 -6
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +446 -221
- data/lib/sidekiq/cli.rb +112 -63
- data/lib/sidekiq/client.rb +57 -60
- data/lib/sidekiq/{util.rb → component.rb} +12 -16
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +48 -37
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +19 -23
- data/lib/sidekiq/job_retry.rb +100 -67
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +145 -59
- data/lib/sidekiq/logger.rb +99 -12
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +99 -44
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +4 -19
- data/lib/sidekiq/paginator.rb +13 -8
- data/lib/sidekiq/processor.rb +64 -60
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +91 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +93 -28
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +51 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +57 -34
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +77 -36
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +41 -73
- data/lib/sidekiq/worker.rb +144 -21
- data/lib/sidekiq.rb +129 -32
- data/sidekiq.gemspec +14 -7
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +52 -69
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +12 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +57 -58
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -47
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq.rb
CHANGED
@@ -5,7 +5,9 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
5
5
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
|
+
require "sidekiq/transaction_aware_client"
|
8
9
|
require "sidekiq/worker"
|
10
|
+
require "sidekiq/job"
|
9
11
|
require "sidekiq/redis_connection"
|
10
12
|
require "sidekiq/delay"
|
11
13
|
|
@@ -20,26 +22,26 @@ module Sidekiq
|
|
20
22
|
labels: [],
|
21
23
|
concurrency: 10,
|
22
24
|
require: ".",
|
25
|
+
strict: true,
|
23
26
|
environment: nil,
|
24
27
|
timeout: 25,
|
25
28
|
poll_interval_average: nil,
|
26
29
|
average_scheduled_poll_interval: 5,
|
30
|
+
on_complex_arguments: :warn,
|
27
31
|
error_handlers: [],
|
28
32
|
death_handlers: [],
|
29
33
|
lifecycle_events: {
|
30
34
|
startup: [],
|
31
35
|
quiet: [],
|
32
36
|
shutdown: [],
|
37
|
+
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
33
38
|
heartbeat: [],
|
39
|
+
# triggers on EVERY heartbeat call, every 10 seconds
|
40
|
+
beat: []
|
34
41
|
},
|
35
42
|
dead_max_jobs: 10_000,
|
36
43
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
37
|
-
reloader: proc { |&block| block.call }
|
38
|
-
}
|
39
|
-
|
40
|
-
DEFAULT_WORKER_OPTIONS = {
|
41
|
-
"retry" => true,
|
42
|
-
"queue" => "default",
|
44
|
+
reloader: proc { |&block| block.call }
|
43
45
|
}
|
44
46
|
|
45
47
|
FAKE_INFO = {
|
@@ -47,26 +49,91 @@ module Sidekiq
|
|
47
49
|
"uptime_in_days" => "9999",
|
48
50
|
"connected_clients" => "9999",
|
49
51
|
"used_memory_human" => "9P",
|
50
|
-
"used_memory_peak_human" => "9P"
|
52
|
+
"used_memory_peak_human" => "9P"
|
51
53
|
}
|
52
54
|
|
53
55
|
def self.❨╯°□°❩╯︵┻━┻
|
54
56
|
puts "Calm down, yo."
|
55
57
|
end
|
56
58
|
|
59
|
+
# config.concurrency = 5
|
60
|
+
def self.concurrency=(val)
|
61
|
+
self[:concurrency] = Integer(val)
|
62
|
+
end
|
63
|
+
|
64
|
+
# config.queues = %w( high default low ) # strict
|
65
|
+
# config.queues = %w( high,3 default,2 low,1 ) # weighted
|
66
|
+
# config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
|
67
|
+
#
|
68
|
+
# With weighted priority, queue will be checked first (weight / total) of the time.
|
69
|
+
# high will be checked first (3/6) or 50% of the time.
|
70
|
+
# I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
|
71
|
+
# are ridiculous and unnecessarily expensive. You can get random queue ordering
|
72
|
+
# by explicitly setting all weights to 1.
|
73
|
+
def self.queues=(val)
|
74
|
+
self[:queues] = Array(val).each_with_object([]) do |qstr, memo|
|
75
|
+
name, weight = qstr.split(",")
|
76
|
+
self[:strict] = false if weight.to_i > 0
|
77
|
+
[weight.to_i, 1].max.times do
|
78
|
+
memo << name
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
### Private APIs
|
84
|
+
def self.default_error_handler(ex, ctx)
|
85
|
+
logger.warn(dump_json(ctx)) unless ctx.empty?
|
86
|
+
logger.warn("#{ex.class.name}: #{ex.message}")
|
87
|
+
logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
88
|
+
end
|
89
|
+
|
90
|
+
# DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
|
91
|
+
# be referenced. It must be defined here, after the default_error_handler
|
92
|
+
# method is defined.
|
93
|
+
DEFAULT_ERROR_HANDLER = method(:default_error_handler)
|
94
|
+
|
95
|
+
@config = DEFAULTS.dup
|
57
96
|
def self.options
|
58
|
-
|
97
|
+
logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
|
98
|
+
@config
|
59
99
|
end
|
60
100
|
|
61
101
|
def self.options=(opts)
|
62
|
-
|
102
|
+
logger.warn "config.options = hash` is deprecated, use `config.merge!(hash)`: #{caller(1..2)}"
|
103
|
+
@config = opts
|
104
|
+
end
|
105
|
+
|
106
|
+
def self.[](key)
|
107
|
+
@config[key]
|
108
|
+
end
|
109
|
+
|
110
|
+
def self.[]=(key, val)
|
111
|
+
@config[key] = val
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.merge!(hash)
|
115
|
+
@config.merge!(hash)
|
63
116
|
end
|
64
117
|
|
118
|
+
def self.fetch(*args, &block)
|
119
|
+
@config.fetch(*args, &block)
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.handle_exception(ex, ctx = {})
|
123
|
+
self[:error_handlers].each do |handler|
|
124
|
+
handler.call(ex, ctx)
|
125
|
+
rescue => ex
|
126
|
+
logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
127
|
+
logger.error ex
|
128
|
+
logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
###
|
132
|
+
|
65
133
|
##
|
66
134
|
# Configuration for Sidekiq server, use like:
|
67
135
|
#
|
68
136
|
# Sidekiq.configure_server do |config|
|
69
|
-
# config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
|
70
137
|
# config.server_middleware do |chain|
|
71
138
|
# chain.add MyServerHook
|
72
139
|
# end
|
@@ -79,7 +146,7 @@ module Sidekiq
|
|
79
146
|
# Configuration for Sidekiq client, use like:
|
80
147
|
#
|
81
148
|
# Sidekiq.configure_client do |config|
|
82
|
-
# config.redis = { :
|
149
|
+
# config.redis = { size: 1, url: 'redis://myhost:8877/0' }
|
83
150
|
# end
|
84
151
|
def self.configure_client
|
85
152
|
yield self unless server?
|
@@ -95,10 +162,13 @@ module Sidekiq
|
|
95
162
|
retryable = true
|
96
163
|
begin
|
97
164
|
yield conn
|
98
|
-
rescue
|
165
|
+
rescue RedisConnection.adapter::BaseError => ex
|
99
166
|
# 2550 Failover can cause the server to become a replica, need
|
100
167
|
# to disconnect and reopen the socket to get back to the primary.
|
101
|
-
if
|
168
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
169
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
170
|
+
# The same retry logic is also used in client.rb
|
171
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
102
172
|
conn.disconnect!
|
103
173
|
retryable = false
|
104
174
|
retry
|
@@ -117,7 +187,7 @@ module Sidekiq
|
|
117
187
|
else
|
118
188
|
conn.info
|
119
189
|
end
|
120
|
-
rescue
|
190
|
+
rescue RedisConnection.adapter::CommandError => ex
|
121
191
|
# 2850 return fake version when INFO command has (probably) been renamed
|
122
192
|
raise unless /unknown command/.match?(ex.message)
|
123
193
|
FAKE_INFO
|
@@ -125,19 +195,19 @@ module Sidekiq
|
|
125
195
|
end
|
126
196
|
|
127
197
|
def self.redis_pool
|
128
|
-
@redis ||=
|
198
|
+
@redis ||= RedisConnection.create
|
129
199
|
end
|
130
200
|
|
131
201
|
def self.redis=(hash)
|
132
202
|
@redis = if hash.is_a?(ConnectionPool)
|
133
203
|
hash
|
134
204
|
else
|
135
|
-
|
205
|
+
RedisConnection.create(hash)
|
136
206
|
end
|
137
207
|
end
|
138
208
|
|
139
209
|
def self.client_middleware
|
140
|
-
@client_chain ||= Middleware::Chain.new
|
210
|
+
@client_chain ||= Middleware::Chain.new(self)
|
141
211
|
yield @client_chain if block_given?
|
142
212
|
@client_chain
|
143
213
|
end
|
@@ -149,16 +219,23 @@ module Sidekiq
|
|
149
219
|
end
|
150
220
|
|
151
221
|
def self.default_server_middleware
|
152
|
-
Middleware::Chain.new
|
222
|
+
Middleware::Chain.new(self)
|
223
|
+
end
|
224
|
+
|
225
|
+
def self.default_worker_options=(hash) # deprecated
|
226
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
227
|
+
end
|
228
|
+
|
229
|
+
def self.default_job_options=(hash)
|
230
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
153
231
|
end
|
154
232
|
|
155
|
-
def self.default_worker_options
|
156
|
-
|
157
|
-
@default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
|
233
|
+
def self.default_worker_options # deprecated
|
234
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
158
235
|
end
|
159
236
|
|
160
|
-
def self.
|
161
|
-
|
237
|
+
def self.default_job_options
|
238
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
162
239
|
end
|
163
240
|
|
164
241
|
##
|
@@ -171,7 +248,7 @@ module Sidekiq
|
|
171
248
|
# end
|
172
249
|
# end
|
173
250
|
def self.death_handlers
|
174
|
-
|
251
|
+
self[:death_handlers]
|
175
252
|
end
|
176
253
|
|
177
254
|
def self.load_json(string)
|
@@ -192,23 +269,39 @@ module Sidekiq
|
|
192
269
|
|
193
270
|
def self.log_formatter=(log_formatter)
|
194
271
|
@log_formatter = log_formatter
|
272
|
+
logger.formatter = log_formatter
|
195
273
|
end
|
196
274
|
|
197
275
|
def self.logger
|
198
|
-
@logger ||= Sidekiq::Logger.new(
|
276
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: :info)
|
199
277
|
end
|
200
278
|
|
201
279
|
def self.logger=(logger)
|
280
|
+
if logger.nil?
|
281
|
+
self.logger.level = Logger::FATAL
|
282
|
+
return self.logger
|
283
|
+
end
|
284
|
+
|
285
|
+
logger.extend(Sidekiq::LoggingUtils)
|
286
|
+
|
202
287
|
@logger = logger
|
203
288
|
end
|
204
289
|
|
290
|
+
def self.pro?
|
291
|
+
defined?(Sidekiq::Pro)
|
292
|
+
end
|
293
|
+
|
294
|
+
def self.ent?
|
295
|
+
defined?(Sidekiq::Enterprise)
|
296
|
+
end
|
297
|
+
|
205
298
|
# How frequently Redis should be checked by a random Sidekiq process for
|
206
299
|
# scheduled and retriable jobs. Each individual process will take turns by
|
207
300
|
# waiting some multiple of this value.
|
208
301
|
#
|
209
302
|
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
210
303
|
def self.average_scheduled_poll_interval=(interval)
|
211
|
-
|
304
|
+
self[:average_scheduled_poll_interval] = interval
|
212
305
|
end
|
213
306
|
|
214
307
|
# Register a proc to handle any error which occurs within the Sidekiq process.
|
@@ -219,7 +312,7 @@ module Sidekiq
|
|
219
312
|
#
|
220
313
|
# The default error handler logs errors to Sidekiq.logger.
|
221
314
|
def self.error_handlers
|
222
|
-
|
315
|
+
self[:error_handlers]
|
223
316
|
end
|
224
317
|
|
225
318
|
# Register a block to run at a point in the Sidekiq lifecycle.
|
@@ -232,16 +325,20 @@ module Sidekiq
|
|
232
325
|
# end
|
233
326
|
def self.on(event, &block)
|
234
327
|
raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
|
235
|
-
raise ArgumentError, "Invalid event name: #{event}" unless
|
236
|
-
|
328
|
+
raise ArgumentError, "Invalid event name: #{event}" unless self[:lifecycle_events].key?(event)
|
329
|
+
self[:lifecycle_events][event] << block
|
330
|
+
end
|
331
|
+
|
332
|
+
def self.strict_args!(mode = :raise)
|
333
|
+
self[:on_complex_arguments] = mode
|
237
334
|
end
|
238
335
|
|
239
|
-
# We are shutting down Sidekiq but what about
|
336
|
+
# We are shutting down Sidekiq but what about threads that
|
240
337
|
# are working on some long job? This error is
|
241
|
-
# raised in
|
338
|
+
# raised in jobs that have not finished within the hard
|
242
339
|
# timeout limit. This is needed to rollback db transactions,
|
243
340
|
# otherwise Ruby's Thread#kill will commit. See #377.
|
244
|
-
# DO NOT RESCUE THIS ERROR IN YOUR
|
341
|
+
# DO NOT RESCUE THIS ERROR IN YOUR JOBS
|
245
342
|
class Shutdown < Interrupt; end
|
246
343
|
end
|
247
344
|
|
data/sidekiq.gemspec
CHANGED
@@ -5,17 +5,24 @@ Gem::Specification.new do |gem|
|
|
5
5
|
gem.email = ["mperham@gmail.com"]
|
6
6
|
gem.summary = "Simple, efficient background processing for Ruby"
|
7
7
|
gem.description = "Simple, efficient background processing for Ruby."
|
8
|
-
gem.homepage = "
|
8
|
+
gem.homepage = "https://sidekiq.org"
|
9
9
|
gem.license = "LGPL-3.0"
|
10
10
|
|
11
|
-
gem.executables = ["sidekiq"]
|
12
|
-
gem.files = `git ls-files | grep -
|
11
|
+
gem.executables = ["sidekiq", "sidekiqmon"]
|
12
|
+
gem.files = ["sidekiq.gemspec", "README.md", "Changes.md", "LICENSE"] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
|
13
13
|
gem.name = "sidekiq"
|
14
14
|
gem.version = Sidekiq::VERSION
|
15
15
|
gem.required_ruby_version = ">= 2.5.0"
|
16
16
|
|
17
|
-
gem.
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
gem.metadata = {
|
18
|
+
"homepage_uri" => "https://sidekiq.org",
|
19
|
+
"bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
|
20
|
+
"documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
|
21
|
+
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
|
22
|
+
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
|
+
}
|
24
|
+
|
25
|
+
gem.add_dependency "redis", "<5", ">= 4.5.0"
|
26
|
+
gem.add_dependency "connection_pool", ">= 2.2.5"
|
27
|
+
gem.add_dependency "rack", "~> 2.0"
|
21
28
|
end
|
Binary file
|