sidekiq 6.4.0 → 6.5.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +50 -1
- data/README.md +6 -1
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/sidekiq/api.rb +78 -65
- data/lib/sidekiq/cli.rb +46 -37
- data/lib/sidekiq/client.rb +42 -28
- data/lib/sidekiq/component.rb +64 -0
- data/lib/sidekiq/delay.rb +1 -1
- data/lib/sidekiq/extensions/action_mailer.rb +2 -2
- data/lib/sidekiq/extensions/active_record.rb +2 -2
- data/lib/sidekiq/extensions/class_methods.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
- data/lib/sidekiq/fetch.rb +17 -15
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +27 -26
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +54 -52
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/middleware/chain.rb +22 -13
- data/lib/sidekiq/middleware/current_attributes.rb +4 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +19 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +81 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +11 -10
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +5 -5
- data/lib/sidekiq/web.rb +3 -3
- data/lib/sidekiq/worker.rb +20 -17
- data/lib/sidekiq.rb +98 -30
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/stylesheets/application.css +1 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +3 -3
- metadata +8 -5
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,23 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
12
13
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
14
|
|
15
15
|
module Sidekiq
|
16
16
|
class CLI
|
17
|
-
include
|
17
|
+
include Sidekiq::Component
|
18
18
|
include Singleton unless $TESTING
|
19
19
|
|
20
20
|
attr_accessor :launcher
|
21
21
|
attr_accessor :environment
|
22
|
+
attr_accessor :config
|
23
|
+
|
24
|
+
def parse(args = ARGV.dup)
|
25
|
+
@config = Sidekiq
|
26
|
+
@config[:error_handlers].clear
|
27
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
22
28
|
|
23
|
-
def parse(args = ARGV)
|
24
29
|
setup_options(args)
|
25
30
|
initialize_logger
|
26
31
|
validate!
|
@@ -36,7 +41,7 @@ module Sidekiq
|
|
36
41
|
def run(boot_app: true)
|
37
42
|
boot_application if boot_app
|
38
43
|
|
39
|
-
if environment == "development" && $stdout.tty? &&
|
44
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
40
45
|
print_banner
|
41
46
|
end
|
42
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -67,7 +72,7 @@ module Sidekiq
|
|
67
72
|
|
68
73
|
# touch the connection pool so it is created before we
|
69
74
|
# fire startup and start multithreading.
|
70
|
-
info =
|
75
|
+
info = @config.redis_info
|
71
76
|
ver = info["redis_version"]
|
72
77
|
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
73
78
|
|
@@ -85,22 +90,22 @@ module Sidekiq
|
|
85
90
|
|
86
91
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
87
92
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
88
|
-
cursize =
|
89
|
-
needed =
|
93
|
+
cursize = @config.redis_pool.size
|
94
|
+
needed = @config[:concurrency] + 2
|
90
95
|
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
91
96
|
|
92
97
|
# cache process identity
|
93
|
-
|
98
|
+
@config[:identity] = identity
|
94
99
|
|
95
100
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
96
|
-
|
101
|
+
@config.server_middleware
|
97
102
|
|
98
103
|
# Before this point, the process is initializing with just the main thread.
|
99
104
|
# Starting here the process will now have multiple threads running.
|
100
105
|
fire_event(:startup, reverse: false, reraise: true)
|
101
106
|
|
102
|
-
logger.debug { "Client Middleware: #{
|
103
|
-
logger.debug { "Server Middleware: #{
|
107
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
104
109
|
|
105
110
|
launch(self_read)
|
106
111
|
end
|
@@ -110,13 +115,13 @@ module Sidekiq
|
|
110
115
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
111
116
|
end
|
112
117
|
|
113
|
-
@launcher = Sidekiq::Launcher.new(
|
118
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
114
119
|
|
115
120
|
begin
|
116
121
|
launcher.run
|
117
122
|
|
118
|
-
while
|
119
|
-
signal =
|
123
|
+
while self_read.wait_readable
|
124
|
+
signal = self_read.gets.strip
|
120
125
|
handle_signal(signal)
|
121
126
|
end
|
122
127
|
rescue Interrupt
|
@@ -173,25 +178,25 @@ module Sidekiq
|
|
173
178
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
174
179
|
"TERM" => ->(cli) { raise Interrupt },
|
175
180
|
"TSTP" => ->(cli) {
|
176
|
-
|
181
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
177
182
|
cli.launcher.quiet
|
178
183
|
},
|
179
184
|
"TTIN" => ->(cli) {
|
180
185
|
Thread.list.each do |thread|
|
181
|
-
|
186
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
182
187
|
if thread.backtrace
|
183
|
-
|
188
|
+
cli.logger.warn thread.backtrace.join("\n")
|
184
189
|
else
|
185
|
-
|
190
|
+
cli.logger.warn "<no backtrace available>"
|
186
191
|
end
|
187
192
|
end
|
188
193
|
}
|
189
194
|
}
|
190
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
195
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
191
196
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
192
197
|
|
193
198
|
def handle_signal(sig)
|
194
|
-
|
199
|
+
logger.debug "Got #{sig} signal"
|
195
200
|
SIGNAL_HANDLERS[sig].call(self)
|
196
201
|
end
|
197
202
|
|
@@ -237,7 +242,7 @@ module Sidekiq
|
|
237
242
|
config_dir = if File.directory?(opts[:require].to_s)
|
238
243
|
File.join(opts[:require], "config")
|
239
244
|
else
|
240
|
-
File.join(
|
245
|
+
File.join(@config[:require], "config")
|
241
246
|
end
|
242
247
|
|
243
248
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -254,27 +259,23 @@ module Sidekiq
|
|
254
259
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
255
260
|
|
256
261
|
# merge with defaults
|
257
|
-
|
258
|
-
end
|
259
|
-
|
260
|
-
def options
|
261
|
-
Sidekiq.options
|
262
|
+
@config.merge!(opts)
|
262
263
|
end
|
263
264
|
|
264
265
|
def boot_application
|
265
266
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
266
267
|
|
267
|
-
if File.directory?(
|
268
|
+
if File.directory?(@config[:require])
|
268
269
|
require "rails"
|
269
270
|
if ::Rails::VERSION::MAJOR < 5
|
270
271
|
raise "Sidekiq no longer supports this version of Rails"
|
271
272
|
else
|
272
273
|
require "sidekiq/rails"
|
273
|
-
require File.expand_path("#{
|
274
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
274
275
|
end
|
275
|
-
|
276
|
+
@config[:tag] ||= default_tag
|
276
277
|
else
|
277
|
-
require
|
278
|
+
require @config[:require]
|
278
279
|
end
|
279
280
|
end
|
280
281
|
|
@@ -291,18 +292,18 @@ module Sidekiq
|
|
291
292
|
end
|
292
293
|
|
293
294
|
def validate!
|
294
|
-
if !File.exist?(
|
295
|
-
(File.directory?(
|
295
|
+
if !File.exist?(@config[:require]) ||
|
296
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
296
297
|
logger.info "=================================================================="
|
297
298
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
298
|
-
logger.info " to load your
|
299
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
299
300
|
logger.info "=================================================================="
|
300
301
|
logger.info @parser
|
301
302
|
die(1)
|
302
303
|
end
|
303
304
|
|
304
305
|
[:concurrency, :timeout].each do |opt|
|
305
|
-
raise ArgumentError, "#{opt}: #{
|
306
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
306
307
|
end
|
307
308
|
end
|
308
309
|
|
@@ -336,7 +337,7 @@ module Sidekiq
|
|
336
337
|
parse_queue opts, queue, weight
|
337
338
|
end
|
338
339
|
|
339
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
340
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
340
341
|
opts[:require] = arg
|
341
342
|
end
|
342
343
|
|
@@ -376,13 +377,13 @@ module Sidekiq
|
|
376
377
|
end
|
377
378
|
|
378
379
|
def initialize_logger
|
379
|
-
|
380
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
380
381
|
end
|
381
382
|
|
382
383
|
def parse_config(path)
|
383
384
|
erb = ERB.new(File.read(path))
|
384
385
|
erb.filename = File.expand_path(path)
|
385
|
-
opts =
|
386
|
+
opts = load_yaml(erb.result) || {}
|
386
387
|
|
387
388
|
if opts.respond_to? :deep_symbolize_keys!
|
388
389
|
opts.deep_symbolize_keys!
|
@@ -398,6 +399,14 @@ module Sidekiq
|
|
398
399
|
opts
|
399
400
|
end
|
400
401
|
|
402
|
+
def load_yaml(src)
|
403
|
+
if Psych::VERSION > "4.0"
|
404
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
405
|
+
else
|
406
|
+
YAML.load(src)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
401
410
|
def parse_queues(opts, queues_and_weights)
|
402
411
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
403
412
|
end
|
data/lib/sidekiq/client.rb
CHANGED
@@ -15,7 +15,7 @@ module Sidekiq
|
|
15
15
|
# client.middleware do |chain|
|
16
16
|
# chain.use MyClientMiddleware
|
17
17
|
# end
|
18
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
19
19
|
#
|
20
20
|
# All client instances default to the globally-defined
|
21
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
@@ -49,16 +49,16 @@ module Sidekiq
|
|
49
49
|
# The main method used to push a job to Redis. Accepts a number of options:
|
50
50
|
#
|
51
51
|
# queue - the named queue to use, default 'default'
|
52
|
-
# class - the
|
52
|
+
# class - the job class to call, required
|
53
53
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
54
54
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
55
55
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
56
56
|
# backtrace - whether to save any error backtrace, default false
|
57
57
|
#
|
58
58
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
59
|
-
#
|
59
|
+
# job options. Otherwise, they will be based on the job class's options.
|
60
60
|
#
|
61
|
-
# Any options valid for a
|
61
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
62
62
|
#
|
63
63
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
64
64
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -67,13 +67,15 @@ module Sidekiq
|
|
67
67
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
68
68
|
#
|
69
69
|
# Example:
|
70
|
-
# push('queue' => 'my_queue', 'class' =>
|
70
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
71
71
|
#
|
72
72
|
def push(item)
|
73
73
|
normed = normalize_item(item)
|
74
|
-
payload =
|
75
|
-
|
74
|
+
payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
|
75
|
+
normed
|
76
|
+
end
|
76
77
|
if payload
|
78
|
+
verify_json(payload)
|
77
79
|
raw_push([payload])
|
78
80
|
payload["jid"]
|
79
81
|
end
|
@@ -101,12 +103,17 @@ module Sidekiq
|
|
101
103
|
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
102
104
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
103
105
|
|
106
|
+
jid = items.delete("jid")
|
107
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
108
|
+
|
104
109
|
normed = normalize_item(items)
|
105
110
|
payloads = args.map.with_index { |job_args, index|
|
106
|
-
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12)
|
111
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
107
112
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
108
|
-
|
109
|
-
|
113
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
114
|
+
verify_json(copy)
|
115
|
+
copy
|
116
|
+
end
|
110
117
|
result || nil
|
111
118
|
}.compact
|
112
119
|
|
@@ -119,8 +126,8 @@ module Sidekiq
|
|
119
126
|
#
|
120
127
|
# pool = ConnectionPool.new { Redis.new }
|
121
128
|
# Sidekiq::Client.via(pool) do
|
122
|
-
#
|
123
|
-
#
|
129
|
+
# SomeJob.perform_async(1,2,3)
|
130
|
+
# SomeOtherJob.perform_async(1,2,3)
|
124
131
|
# end
|
125
132
|
#
|
126
133
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -145,10 +152,10 @@ module Sidekiq
|
|
145
152
|
end
|
146
153
|
|
147
154
|
# Resque compatibility helpers. Note all helpers
|
148
|
-
# should go through
|
155
|
+
# should go through Sidekiq::Job#client_push.
|
149
156
|
#
|
150
157
|
# Example usage:
|
151
|
-
# Sidekiq::Client.enqueue(
|
158
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
152
159
|
#
|
153
160
|
# Messages are enqueued to the 'default' queue.
|
154
161
|
#
|
@@ -157,14 +164,14 @@ module Sidekiq
|
|
157
164
|
end
|
158
165
|
|
159
166
|
# Example usage:
|
160
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
167
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
161
168
|
#
|
162
169
|
def enqueue_to(queue, klass, *args)
|
163
170
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
164
171
|
end
|
165
172
|
|
166
173
|
# Example usage:
|
167
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
174
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
168
175
|
#
|
169
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
170
177
|
int = interval.to_f
|
@@ -178,7 +185,7 @@ module Sidekiq
|
|
178
185
|
end
|
179
186
|
|
180
187
|
# Example usage:
|
181
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
188
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
182
189
|
#
|
183
190
|
def enqueue_in(interval, klass, *args)
|
184
191
|
klass.perform_in(interval, *args)
|
@@ -189,8 +196,23 @@ module Sidekiq
|
|
189
196
|
|
190
197
|
def raw_push(payloads)
|
191
198
|
@redis_pool.with do |conn|
|
192
|
-
|
193
|
-
|
199
|
+
retryable = true
|
200
|
+
begin
|
201
|
+
conn.pipelined do |pipeline|
|
202
|
+
atomic_push(pipeline, payloads)
|
203
|
+
end
|
204
|
+
rescue RedisConnection.adapter::BaseError => ex
|
205
|
+
# 2550 Failover can cause the server to become a replica, need
|
206
|
+
# to disconnect and reopen the socket to get back to the primary.
|
207
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
|
+
# The retry logic is copied from sidekiq.rb
|
210
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
+
conn.disconnect!
|
212
|
+
retryable = false
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
raise
|
194
216
|
end
|
195
217
|
end
|
196
218
|
true
|
@@ -198,7 +220,7 @@ module Sidekiq
|
|
198
220
|
|
199
221
|
def atomic_push(conn, payloads)
|
200
222
|
if payloads.first.key?("at")
|
201
|
-
conn.zadd("schedule", payloads.map { |hash|
|
223
|
+
conn.zadd("schedule", *payloads.map { |hash|
|
202
224
|
at = hash.delete("at").to_s
|
203
225
|
[at, Sidekiq.dump_json(hash)]
|
204
226
|
})
|
@@ -213,13 +235,5 @@ module Sidekiq
|
|
213
235
|
conn.lpush("queue:#{queue}", to_push)
|
214
236
|
end
|
215
237
|
end
|
216
|
-
|
217
|
-
def process_single(worker_class, item)
|
218
|
-
queue = item["queue"]
|
219
|
-
|
220
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
221
|
-
item
|
222
|
-
end
|
223
|
-
end
|
224
238
|
end
|
225
239
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
##
|
3
|
+
# Sidekiq::Component assumes a config instance is available at @config
|
4
|
+
module Component
|
5
|
+
attr_reader :config
|
6
|
+
|
7
|
+
def watchdog(last_words)
|
8
|
+
yield
|
9
|
+
rescue Exception => ex
|
10
|
+
handle_exception(ex, {context: last_words})
|
11
|
+
raise ex
|
12
|
+
end
|
13
|
+
|
14
|
+
def safe_thread(name, &block)
|
15
|
+
Thread.new do
|
16
|
+
Thread.current.name = name
|
17
|
+
watchdog(name, &block)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def logger
|
22
|
+
config.logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def redis(&block)
|
26
|
+
config.redis(&block)
|
27
|
+
end
|
28
|
+
|
29
|
+
def tid
|
30
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
31
|
+
end
|
32
|
+
|
33
|
+
def hostname
|
34
|
+
ENV["DYNO"] || Socket.gethostname
|
35
|
+
end
|
36
|
+
|
37
|
+
def process_nonce
|
38
|
+
@@process_nonce ||= SecureRandom.hex(6)
|
39
|
+
end
|
40
|
+
|
41
|
+
def identity
|
42
|
+
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def handle_exception(ex, ctx = {})
|
46
|
+
config.handle_exception(ex, ctx)
|
47
|
+
end
|
48
|
+
|
49
|
+
def fire_event(event, options = {})
|
50
|
+
reverse = options[:reverse]
|
51
|
+
reraise = options[:reraise]
|
52
|
+
|
53
|
+
arr = config[:lifecycle_events][event]
|
54
|
+
arr.reverse! if reverse
|
55
|
+
arr.each do |block|
|
56
|
+
block.call
|
57
|
+
rescue => ex
|
58
|
+
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
59
|
+
raise ex if reraise
|
60
|
+
end
|
61
|
+
arr.clear # once we've fired an event, we never fire it again
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Sidekiq
|
4
4
|
module Extensions
|
5
5
|
def self.enable_delay!
|
6
|
-
|
6
|
+
warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
|
7
7
|
|
8
8
|
if defined?(::ActiveSupport)
|
9
9
|
require "sidekiq/extensions/active_record"
|
@@ -16,8 +16,8 @@ module Sidekiq
|
|
16
16
|
include Sidekiq::Worker
|
17
17
|
|
18
18
|
def perform(yml)
|
19
|
-
(target, method_name, args
|
20
|
-
msg =
|
19
|
+
(target, method_name, args) = YAML.load(yml)
|
20
|
+
msg = target.public_send(method_name, *args)
|
21
21
|
# The email method can return nil, which causes ActionMailer to return
|
22
22
|
# an undeliverable empty message.
|
23
23
|
if msg
|
@@ -18,8 +18,8 @@ module Sidekiq
|
|
18
18
|
include Sidekiq::Worker
|
19
19
|
|
20
20
|
def perform(yml)
|
21
|
-
(target, method_name, args
|
22
|
-
|
21
|
+
(target, method_name, args) = YAML.load(yml)
|
22
|
+
target.__send__(method_name, *args)
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
@@ -16,8 +16,8 @@ module Sidekiq
|
|
16
16
|
include Sidekiq::Worker
|
17
17
|
|
18
18
|
def perform(yml)
|
19
|
-
(target, method_name, args
|
20
|
-
|
19
|
+
(target, method_name, args) = YAML.load(yml)
|
20
|
+
target.__send__(method_name, *args)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
@@ -10,16 +10,16 @@ module Sidekiq
|
|
10
10
|
def initialize(performable, target, options = {})
|
11
11
|
@performable = performable
|
12
12
|
@target = target
|
13
|
-
@opts = options
|
13
|
+
@opts = options.transform_keys(&:to_s)
|
14
14
|
end
|
15
15
|
|
16
|
-
def method_missing(name, *args
|
16
|
+
def method_missing(name, *args)
|
17
17
|
# Sidekiq has a limitation in that its message must be JSON.
|
18
18
|
# JSON can't round trip real Ruby objects so we use YAML to
|
19
19
|
# serialize the objects to a String. The YAML will be converted
|
20
20
|
# to JSON and then deserialized on the other side back into a
|
21
21
|
# Ruby object.
|
22
|
-
obj = [@target, name, args
|
22
|
+
obj = [@target, name, args]
|
23
23
|
marshalled = ::YAML.dump(obj)
|
24
24
|
if marshalled.size > SIZE_LIMIT
|
25
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
+
require "sidekiq/component"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class BasicFetch
|
8
|
+
include Sidekiq::Component
|
7
9
|
# We want the fetch operation to timeout every few seconds so the thread
|
8
10
|
# can check if the process is shutting down.
|
9
11
|
TIMEOUT = 2
|
10
12
|
|
11
|
-
UnitOfWork = Struct.new(:queue, :job) {
|
13
|
+
UnitOfWork = Struct.new(:queue, :job, :config) {
|
12
14
|
def acknowledge
|
13
15
|
# nothing to do
|
14
16
|
end
|
@@ -18,17 +20,17 @@ module Sidekiq
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def requeue
|
21
|
-
|
23
|
+
config.redis do |conn|
|
22
24
|
conn.rpush(queue, job)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
}
|
26
28
|
|
27
|
-
def initialize(
|
28
|
-
raise ArgumentError, "missing queue list" unless
|
29
|
-
@
|
30
|
-
@strictly_ordered_queues = !!@
|
31
|
-
@queues = @
|
29
|
+
def initialize(config)
|
30
|
+
raise ArgumentError, "missing queue list" unless config[:queues]
|
31
|
+
@config = config
|
32
|
+
@strictly_ordered_queues = !!@config[:strict]
|
33
|
+
@queues = @config[:queues].map { |q| "queue:#{q}" }
|
32
34
|
if @strictly_ordered_queues
|
33
35
|
@queues.uniq!
|
34
36
|
@queues << TIMEOUT
|
@@ -44,30 +46,30 @@ module Sidekiq
|
|
44
46
|
return nil
|
45
47
|
end
|
46
48
|
|
47
|
-
|
48
|
-
UnitOfWork.new(
|
49
|
+
queue, job = redis { |conn| conn.brpop(*qs) }
|
50
|
+
UnitOfWork.new(queue, job, config) if queue
|
49
51
|
end
|
50
52
|
|
51
53
|
def bulk_requeue(inprogress, options)
|
52
54
|
return if inprogress.empty?
|
53
55
|
|
54
|
-
|
56
|
+
logger.debug { "Re-queueing terminated jobs" }
|
55
57
|
jobs_to_requeue = {}
|
56
58
|
inprogress.each do |unit_of_work|
|
57
59
|
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
60
|
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
59
61
|
end
|
60
62
|
|
61
|
-
|
62
|
-
conn.pipelined do
|
63
|
+
redis do |conn|
|
64
|
+
conn.pipelined do |pipeline|
|
63
65
|
jobs_to_requeue.each do |queue, jobs|
|
64
|
-
|
66
|
+
pipeline.rpush(queue, jobs)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
68
|
-
|
70
|
+
logger.info("Pushed #{inprogress.size} jobs back to Redis")
|
69
71
|
rescue => ex
|
70
|
-
|
72
|
+
logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
71
73
|
end
|
72
74
|
|
73
75
|
# Creating the Redis#brpop command takes into account any
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -12,46 +12,34 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
yield
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
end
|
15
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
16
|
+
@logger.info("done")
|
18
17
|
rescue Exception
|
19
|
-
|
20
|
-
|
21
|
-
end
|
18
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
19
|
+
@logger.info("fail")
|
22
20
|
|
23
21
|
raise
|
24
22
|
end
|
25
23
|
|
26
24
|
def prepare(job_hash, &block)
|
27
|
-
level = job_hash["log_level"]
|
28
|
-
if level
|
29
|
-
@logger.log_at(level) do
|
30
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
-
end
|
32
|
-
else
|
33
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def job_hash_context(job_hash)
|
38
25
|
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
39
26
|
# attribute to expose the underlying thing.
|
40
27
|
h = {
|
41
28
|
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
|
42
29
|
jid: job_hash["jid"]
|
43
30
|
}
|
44
|
-
h[:bid] = job_hash["bid"] if job_hash
|
45
|
-
h[:tags] = job_hash["tags"] if job_hash
|
46
|
-
h
|
47
|
-
end
|
48
|
-
|
49
|
-
def with_elapsed_time_context(start, &block)
|
50
|
-
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
51
|
-
end
|
31
|
+
h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
|
32
|
+
h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
|
52
33
|
|
53
|
-
|
54
|
-
|
34
|
+
Thread.current[:sidekiq_context] = h
|
35
|
+
level = job_hash["log_level"]
|
36
|
+
if level
|
37
|
+
@logger.log_at(level, &block)
|
38
|
+
else
|
39
|
+
yield
|
40
|
+
end
|
41
|
+
ensure
|
42
|
+
Thread.current[:sidekiq_context] = nil
|
55
43
|
end
|
56
44
|
|
57
45
|
private
|